- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: inithello: Fix for failing upload tool functional tests.
by commits-noreply＠bitbucket.org 25 Apr '13

25 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3d569f107f1d/ Changeset: 3d569f107f1d User: inithello Date: 2013-04-25 21:16:23 Summary: Fix for failing upload tool functional tests. Affected #: 1 file diff -r 1fa287b15af5dab0ac84545835752b4f87e87328 -r 3d569f107f1d449b028fc4d5390dd8d07fdf26ce test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -242,7 +242,20 @@ # Wait for upload processing to finish (TODO: this should be done in each test case instead) self.wait() + def json_from_url( self, url ): + self.visit_url( url ) + return from_json_string( self.last_page() ) + # Functions associated with histories + def get_history_from_api( self, encoded_history_id=None ): + if encoded_history_id is None: + history = self.get_latest_history() + encoded_history_id = history[ 'id' ] + return self.json_from_url( '/api/histories/%s/contents' % encoded_history_id ) + + def get_latest_history( self ): + return self.json_from_url( '/api/histories' )[ 0 ] + def check_history_for_errors( self ): """Raises an exception if there are errors in a history""" self.home() @@ -317,43 +330,29 @@ Uses history page JSON to determine whether this history is empty (i.e. has no undeleted datasets). """ - def has_no_undeleted_hdas( hda_list ): - if not len( hda_list ): - return True - for hda in hda_list: - if not( hda[ 'deleted' ] or hda[ 'purged' ] ): - return False - return True - try: - self.check_history_json( r'\bhdas\s*=\s*(.*);', has_no_undeleted_hdas ) - except AssertionError, exc: - log.error( 'history is not empty' ) - raise exc + return len( self.get_history_from_api() ) == 0 def check_hda_json_for_key_value( self, hda_id, key, value, use_string_contains=False ): """ - Uses history page JSON to determine whether the current history: - (1) has an hda with hda_id, - (2) that hda has a JSON var named 'key', - (3) that var 'key' == value - If use_string_contains=True, this will search for value in var 'key' - instead of testing for an entire, exact match (string only). + Uses the history API to determine whether the current history: + (1) Has a history dataset with the required ID. + (2) That dataset has the required key. + (3) The contents of that key match the provided value. + If use_string_contains=True, this will perform a substring match, otherwise an exact match. """ #TODO: multi key, value - def hda_has_key_value( hda_list ): - for hda in hda_list: - # if we found the hda and there's a var in the json named key - if( ( hda[ 'id' ] == hda_id ) - and ( key in hda ) ): - var = hda[ key ] - # test for partial string containment if str and requested - if( ( type( var ) == str ) - and ( use_string_contains ) ): - return ( value in var ) - # otherwise, test for equivalence - return ( var == value ) - return False - self.check_history_json( r'\bhdas\s*=\s*(.*);', hda_has_key_value ) + hda = dict() + for history_item in self.get_history_from_api(): + if history_item[ 'id' ] == hda_id: + hda = self.json_from_url( history_item[ 'url' ] ) + break + if hda: + if key in hda: + if use_string_contains: + return value in hda[ key ] + else: + return value == hda[ key ] + return False def clear_history( self ): """Empties a history of all datasets""" Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Tools API: ensure error handling on index, show; Browser tests: add api.tools module and create tests for index, show
by commits-noreply＠bitbucket.org 25 Apr '13

25 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1fa287b15af5/ Changeset: 1fa287b15af5 User: carlfeberhard Date: 2013-04-25 20:31:43 Summary: Tools API: ensure error handling on index, show; Browser tests: add api.tools module and create tests for index, show Affected #: 3 files diff -r 6eed572a8bdcd6dc9bf2fcd54360217c1f447f20 -r 1fa287b15af5dab0ac84545835752b4f87e87328 lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -5,6 +5,9 @@ from galaxy.util.json import to_json_string, from_json_string from galaxy.visualization.data_providers.genome import * +import logging +log = logging.getLogger( __name__ ) + class ToolsController( BaseAPIController, UsesVisualizationMixin ): """ RESTful controller for interactions with tools. @@ -29,7 +32,12 @@ trackster = util.string_as_bool( kwds.get( 'trackster', 'False' ) ) # Create return value. - return self.app.toolbox.to_dict( trans, in_panel=in_panel, trackster=trackster ) + try: + return self.app.toolbox.to_dict( trans, in_panel=in_panel, trackster=trackster ) + except Exception, exc: + log.error( 'could not convert toolbox to dictionary: %s', str( exc ), exc_info=True ) + trans.response.status = 500 + return { 'error': str( exc ) } @web.expose_api def show( self, trans, id, **kwd ): @@ -37,7 +45,12 @@ GET /api/tools/{tool_id} Returns tool information, including parameters and inputs. """ - return self.app.toolbox.tools_by_id[ id ].to_dict( trans, for_display=True ) + try: + return self.app.toolbox.tools_by_id[ id ].to_dict( trans, for_display=True ) + except Exception, exc: + log.error( 'could not convert tool (%s) to dictionary: %s', id, str( exc ), exc_info=True ) + trans.response.status = 500 + return { 'error': str( exc ) } @web.expose_api def create( self, trans, payload, **kwd ): @@ -45,7 +58,6 @@ POST /api/tools Executes tool using specified inputs and returns tool's outputs. """ - # HACK: for now, if action is rerun, rerun tool. action = payload.get( 'action', None ) if action == 'rerun': diff -r 6eed572a8bdcd6dc9bf2fcd54360217c1f447f20 -r 1fa287b15af5dab0ac84545835752b4f87e87328 test/casperjs/api-tool-tests.js --- /dev/null +++ b/test/casperjs/api-tool-tests.js @@ -0,0 +1,222 @@ +/* Utility to load a specific page and output html, page text, or a screenshot + * Optionally wait for some time, text, or dom selector + */ +try { + //...if there's a better way - please let me know, universe + var scriptDir = require( 'system' ).args[3] + // remove the script filename + .replace( /[\w|\.|\-|_]*$/, '' ) + // if given rel. path, prepend the curr dir + .replace( /^(?!\/)/, './' ), + spaceghost = require( scriptDir + 'spaceghost' ).create({ + // script options here (can be overridden by CLI) + //verbose: true, + //logLevel: debug, + scriptDir: scriptDir + }); + +} catch( error ){ + console.debug( error ); + phantom.exit( 1 ); +} +spaceghost.start(); + +// =================================================================== SET UP +var utils = require( 'utils' ); + +var email = spaceghost.user.getRandomEmail(), + password = '123456'; +if( spaceghost.fixtureData.testUser ){ + email = spaceghost.fixtureData.testUser.email; + password = spaceghost.fixtureData.testUser.password; +} +spaceghost.user.loginOrRegisterUser( email, password ); + +function hasKeys( object, keysArray ){ + if( !utils.isObject( object ) ){ return false; } + for( var i=0; i<keysArray.length; i += 1 ){ + if( !object.hasOwnProperty( keysArray[i] ) ){ + spaceghost.debug( 'key not found: ' + keysArray[i] ); + return false; + } + } + return true; +} + +function compareObjs( obj1, where ){ + for( var key in where ){ + if( where.hasOwnProperty( key ) ){ + if( !obj1.hasOwnProperty( key ) ){ return false; } + if( obj1[ key ] !== where[ key ] ){ return false; } + } + } + return true; +} + +function findObject( objectArray, where, start ){ + start = start || 0; + for( var i=start; i<objectArray.length; i += 1 ){ + if( compareObjs( objectArray[i], where ) ){ return objectArray[i]; } + } + return null; +} + +// =================================================================== TESTS +var panelSectionKeys = [ + 'elems', 'id', 'name', 'type', 'version' + ], + panelToolKeys = [ + 'id', 'name', 'description', 'version', 'link', 'target', 'min_width', 'type' + ], + toolSummaryKeys = [ + 'id', 'name', 'description', 'version' + ], + toolDetailKeys = [ + 'id', 'name', 'description', 'version', 'inputs' + ], + toolInputKeys = [ + 'html', 'label', 'name', 'type' + // there are others, but it's not consistent across all inputs + ]; + +function attemptShowOnAllTools(){ + //NOTE: execute like: attemptShowOnAllTools.call( spaceghost ) + toolIndex = this.api.tools.index( false ); + var toolErrors = {}; + function ObjectKeySet(){ + var self = this; + function addOne( key ){ + if( !self.hasOwnProperty( key ) ){ + self[ key ] = true; + } + } + self.__add = function( obj ){ + for( var key in obj ){ + if( obj.hasOwnProperty( key ) ){ + addOne( key ); + } + } + }; + return self; + } + var set = new ObjectKeySet(); + for( i=0; i<toolIndex.length; i+=1 ){ + var tool = toolIndex[i]; + try { + toolShow = this.api.tools.show( tool.id ); + this.info( 'checking: ' + tool.id ); + for( var j=0; j<toolShow.inputs.length; j+=1 ){ + var input = toolShow.inputs[j]; + set.__add( input ); + } + } catch( err ){ + var message = JSON.parse( err.message ).error; + this.error( '\t error: ' + message ); + toolErrors[ tool.id ] = message; + } + } + this.debug( this.jsonStr( toolErrors ) ); + this.debug( this.jsonStr( set ) ); +} + +spaceghost.thenOpen( spaceghost.baseUrl ).then( function(){ + + // ------------------------------------------------------------------------------------------- INDEX + // ........................................................................................... (defaults) + this.test.comment( 'index should get a list of tools in panel form (by default)' ); + var toolIndex = this.api.tools.index(); + //this.debug( this.jsonStr( toolIndex ) ); + this.test.assert( utils.isArray( toolIndex ), "index returned an array: length " + toolIndex.length ); + this.test.assert( toolIndex.length >= 1, 'Has at least one tool section' ); + + this.test.comment( 'index panel form should be separated into sections (by default)' ); + var firstSection = toolIndex[0]; // get data + //this.debug( this.jsonStr( firstSection ) ); + this.test.assert( hasKeys( firstSection, panelSectionKeys ), 'Has the proper keys' ); + //TODO: test form of indiv. keys + + this.test.comment( 'index sections have a list of tool "elems"' ); + this.test.assert( utils.isArray( firstSection.elems ), firstSection.name + ".elems is an array: " + + "length " + firstSection.elems.length ); + this.test.assert( firstSection.elems.length >= 1, 'Has at least one tool' ); + + var firstTool = firstSection.elems[0]; // get data + //this.debug( this.jsonStr( firstTool ) ); + this.test.assert( hasKeys( firstTool, panelToolKeys ), 'Has the proper keys' ); + + // ........................................................................................... in_panel=False + this.test.comment( 'index should get a list of all tools when in_panel=false' ); + toolIndex = this.api.tools.index( false ); + //this.debug( this.jsonStr( toolIndex ) ); + this.test.assert( utils.isArray( toolIndex ), "index returned an array: length " + toolIndex.length ); + this.test.assert( toolIndex.length >= 1, 'Has at least one tool' ); + + this.test.comment( 'index non-panel form should be a simple list of tool summaries' ); + firstSection = toolIndex[0]; + //this.debug( this.jsonStr( firstSection ) ); + this.test.assert( hasKeys( firstSection, toolSummaryKeys ), 'Has the proper keys' ); + //TODO: test uniqueness of ids + //TODO: test form of indiv. keys + + // ........................................................................................... trackster=True + this.test.comment( '(like in_panel=True) index with trackster=True should ' + + 'get a (smaller) list of tools in panel form (by default)' ); + toolIndex = this.api.tools.index( undefined, true ); + //this.debug( this.jsonStr( toolIndex ) ); + this.test.assert( utils.isArray( toolIndex ), "index returned an array: length " + toolIndex.length ); + this.test.assert( toolIndex.length >= 1, 'Has at least one tool section' ); + + this.test.comment( 'index with trackster=True should be separated into sections (by default)' ); + firstSection = toolIndex[0]; // get data + //this.debug( this.jsonStr( firstSection ) ); + this.test.assert( hasKeys( firstSection, panelSectionKeys ), 'Has the proper keys' ); + //TODO: test form of indiv. keys + + this.test.comment( 'index sections with trackster=True have a list of tool "elems"' ); + this.test.assert( utils.isArray( firstSection.elems ), firstSection.name + ".elems is an array: " + + "length " + firstSection.elems.length ); + this.test.assert( firstSection.elems.length >= 1, 'Has at least one tool' ); + + firstTool = firstSection.elems[0]; // get data + //this.debug( this.jsonStr( firstTool ) ); + this.test.assert( hasKeys( firstTool, panelToolKeys ), 'Has the proper keys' ); + + // ............................................................................ trackster=True, in_panel=False + // this yields the same as in_panel=False... + + + // ------------------------------------------------------------------------------------------- SHOW + this.test.comment( 'show should get detailed data about the tool with the given id' ); + // get the tool select first from tool index + toolIndex = this.api.tools.index(); + var selectFirst = findObject( findObject( toolIndex, { id: 'textutil' }).elems, { id: 'Show beginning1' }); + //this.debug( this.jsonStr( selectFirst ) ); + + var toolShow = this.api.tools.show( selectFirst.id ); + //this.debug( this.jsonStr( toolShow ) ); + this.test.assert( utils.isObject( toolShow ), "show returned an object" ); + this.test.assert( hasKeys( toolShow, toolDetailKeys ), 'Has the proper keys' ); + + this.test.comment( 'show data should include an array of input objects' ); + this.test.assert( utils.isArray( toolShow.inputs ), "inputs is an array: " + + "length " + toolShow.inputs.length ); + this.test.assert( toolShow.inputs.length >= 1, 'Has at least one element' ); + for( var i=0; i<toolShow.inputs.length; i += 1 ){ + var input = toolShow.inputs[i]; + this.test.comment( 'checking input #' + i + ': ' + ( input.name || '(no name)' ) ); + this.test.assert( utils.isObject( input ), "input is an object" ); + this.test.assert( hasKeys( input, toolInputKeys ), 'Has the proper keys' ); + } + //TODO: test form of indiv. keys + + + // ------------------------------------------------------------------------------------------- CREATE + // this is a method of running a job. Shouldn't that be in jobs.create? + + // ------------------------------------------------------------------------------------------- MISC + //attemptShowOnAllTools.call( spaceghost ); +}); + +// =================================================================== +spaceghost.run( function(){ +}); diff -r 6eed572a8bdcd6dc9bf2fcd54360217c1f447f20 -r 1fa287b15af5dab0ac84545835752b4f87e87328 test/casperjs/modules/api.js --- a/test/casperjs/modules/api.js +++ b/test/casperjs/modules/api.js @@ -20,6 +20,7 @@ this.histories = new HistoriesAPI( this ); this.hdas = new HDAAPI( this ); + this.tools = new ToolsAPI( this ); }; exports.API = API; @@ -219,7 +220,7 @@ }; HDAAPI.prototype.index = function index( historyId, ids ){ - this.api.spaceghost.info( 'hda.index: ' + [ historyId, ids ] ); + this.api.spaceghost.info( 'hdas.index: ' + [ historyId, ids ] ); var data = {}; if( ids ){ ids = ( utils.isArray( ids ) )?( ids.join( ',' ) ):( ids ); @@ -232,7 +233,7 @@ }; HDAAPI.prototype.show = function show( historyId, id, deleted ){ - this.api.spaceghost.info( 'hda.show: ' + [ historyId, id, (( deleted )?( 'w deleted' ):( '' )) ] ); + this.api.spaceghost.info( 'hdas.show: ' + [ historyId, id, (( deleted )?( 'w deleted' ):( '' )) ] ); id = ( id === 'most_recently_used' )?( id ):( this.api.ensureId( id ) ); deleted = deleted || false; @@ -242,7 +243,7 @@ }; HDAAPI.prototype.create = function create( historyId, payload ){ - this.api.spaceghost.info( 'hda.create: ' + [ historyId, this.api.spaceghost.jsonStr( payload ) ] ); + this.api.spaceghost.info( 'hdas.create: ' + [ historyId, this.api.spaceghost.jsonStr( payload ) ] ); // py.payload <-> ajax.data payload = this.api.ensureObject( payload ); @@ -253,7 +254,7 @@ }; HDAAPI.prototype.update = function create( historyId, id, payload ){ - this.api.spaceghost.info( 'hda.update: ' + [ historyId, id, this.api.spaceghost.jsonStr( payload ) ] ); + this.api.spaceghost.info( 'hdas.update: ' + [ historyId, id, this.api.spaceghost.jsonStr( payload ) ] ); // py.payload <-> ajax.data historyId = this.api.ensureId( historyId ); @@ -267,3 +268,53 @@ }); }; +// =================================================================== TOOLS +var ToolsAPI = function HDAAPI( api ){ + this.api = api; +}; +ToolsAPI.prototype.toString = function toString(){ + return this.api + '.ToolsAPI'; +}; + +// ------------------------------------------------------------------- +ToolsAPI.prototype.urlTpls = { + index : 'api/tools', + show : 'api/tools/%s', + create : 'api/tools' +}; + +ToolsAPI.prototype.index = function index( in_panel, trackster ){ + this.api.spaceghost.info( 'tools.index: ' + [ in_panel, trackster ] ); + var data = {}; + // in_panel defaults to true, trackster defaults to false + if( in_panel !== undefined ){ + data.in_panel = ( in_panel )?( true ):( false ); + } + if( in_panel !== undefined ){ + data.trackster = ( trackster )?( true ):( false ); + } + return this.api._ajax( utils.format( this.urlTpls.index ), { + data : data + }); +}; + +ToolsAPI.prototype.show = function show( id ){ + this.api.spaceghost.info( 'tools.show: ' + [ id ] ); + var data = {}; + + return this.api._ajax( utils.format( this.urlTpls.show, id ), { + data : data + }); +}; + +//ToolsAPI.prototype.create = function create( payload ){ +// this.api.spaceghost.info( 'tools.create: ' + [ this.api.spaceghost.jsonStr( payload ) ] ); +// +// // py.payload <-> ajax.data +// payload = this.api.ensureObject( payload ); +// return this.api._ajax( utils.format( this.urlTpls.create, this.api.ensureId( historyId ) ), { +// type : 'POST', +// data : payload +// }); +//}; + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Update tool functional test result mako template to reflect the new structure of the test results dict. Add compatibility with the stable branch to the functional tests' API updater.
by commits-noreply＠bitbucket.org 25 Apr '13

25 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6eed572a8bdc/ Changeset: 6eed572a8bdc User: inithello Date: 2013-04-25 18:23:04 Summary: Update tool functional test result mako template to reflect the new structure of the test results dict. Add compatibility with the stable branch to the functional tests' API updater. Affected #: 2 files diff -r 41d8cdde47297746aa82ce4858006bd2632331db -r 6eed572a8bdcd6dc9bf2fcd54360217c1f447f20 templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako --- a/templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako +++ b/templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako @@ -56,15 +56,15 @@ # repository_metadata.tools_functionally_correct column is set to True. tool_test_results = repository_metadata.tool_test_results test_environment_dict = tool_test_results.get( 'test_environment', None ) - invalid_tests = tool_test_results.get( 'invalid_tests', [] ) - test_errors = tool_test_results.get( 'test_errors', [] ) - tests_passed = tool_test_results.get( 'tests_passed', [] ) + missing_test_components = tool_test_results.get( 'missing_test_components', [] ) + failed_tests = tool_test_results.get( 'failed_tests', [] ) + passed_tests = tool_test_results.get( 'passed_tests', [] ) else: tool_test_results = None test_environment_dict = {} - invalid_tests = [] - test_errors = [] - tests_passed = [] + missing_test_components = [] + failed_tests = [] + passed_tests = [] if can_push: browse_label = 'Browse or delete repository tip files' @@ -144,7 +144,7 @@ <b>Repository name:</b><br/> ${repository.name} %endif -%if invalid_tests or tool_test_results or tests_passed: +%if missing_test_components or tool_test_results or passed_tests: <p/><div class="toolForm"><div class="toolFormTitle">Tool functional test results</div> @@ -203,7 +203,7 @@ ${test_environment_dict.get( 'python_version', 'unknown' ) | h} <div style="clear: both"></div></div> - %if test_errors: + %if failed_tests: <div class="form-row"><table width="100%"><tr bgcolor="#D8D8D8" width="100%"><td><b>Tests that failed</td></tr> @@ -211,7 +211,7 @@ </div><div class="form-row"><table class="grid"> - %for test_results_dict in test_errors: + %for test_results_dict in failed_tests: <% test_id = test_results_dict.get( 'test_id', 'unknown' ) tool_id = test_results_dict.get( 'tool_id', 'unknown' ) @@ -244,7 +244,7 @@ <div style="clear: both"></div></div> %endif - %if tests_passed: + %if passed_tests: <div class="form-row"><table width="100%"><tr bgcolor="#D8D8D8" width="100%"><td><b>Tests that passed successfully</td></tr> @@ -252,7 +252,7 @@ </div><div class="form-row"><table class="grid"> - %for test_results_dict in tests_passed: + %for test_results_dict in passed_tests: <% test_id = test_results_dict.get( 'test_id', 'unknown' ) tool_id = test_results_dict.get( 'tool_id', 'unknown' ) @@ -274,7 +274,7 @@ </table></div> %endif - %if invalid_tests: + %if missing_test_components: <div class="form-row"><table width="100%"><tr bgcolor="#D8D8D8" width="100%"><td><b>Invalid tests</td></tr> @@ -282,22 +282,22 @@ </div><div class="form-row"><table class="grid"> - %for test_results_dict in invalid_tests: + %for test_results_dict in missing_test_components: <% guid = test_results_dict.get( 'tool_guid', None ) tool_id = test_results_dict.get( 'tool_id', None ) tool_version = test_results_dict.get( 'tool_version', None ) - reason_test_is_invalid = test_results_dict.get( 'reason_test_is_invalid', None ) + missing_components = test_results_dict.get( 'missing_components', None ) %> %if tool_id or tool_version: <tr><td colspan="2" bgcolor="#FFFFCC">Tool id: <b>${tool_id}</b> version: <b>${tool_version}</b></td></tr> %endif - %if reason_test_is_invalid: + %if missing_components: <tr><td><b>Reason test is invalid</b></td> - <td>${render_functional_test_text( reason_test_is_invalid )}</td> + <td>${render_functional_test_text( missing_components )}</td></tr> %endif %endfor diff -r 41d8cdde47297746aa82ce4858006bd2632331db -r 6eed572a8bdcd6dc9bf2fcd54360217c1f447f20 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -291,6 +291,17 @@ params[ 'tools_functionally_correct' ] = 'false' params[ 'do_not_test' ] = 'false' params[ 'tool_test_results' ] = test_results_dict + # BEGIN compatibility code. + # TODO: The repository_revisions API controller ignores any received parameter that is not present in the database schema, + # but the compatibility code here should be removed when the main tool shed is updated with the new database migration version. + params[ 'tool_test_errors' ] = test_results_dict + if test_results_dict[ 'failed_tests' ]: + params[ 'tool_test_errors' ][ 'test_errors' ] = test_results_dict[ 'failed_tests' ] + if test_results_dict[ 'passed_tests' ]: + params[ 'tool_test_errors' ][ 'tests_passed' ] = test_results_dict[ 'passed_tests' ] + # Copying the missing_test_components key is not necessary, since the main tool shed is running its own version of the + # check repositories script, which correctly updates the invalid_tests key. + # END compatibility code. if '-info_only' in sys.argv: return {} else: @@ -636,7 +647,8 @@ repository_status[ 'test_environment' ] = test_environment repository_status[ 'passed_tests' ] = [] repository_status[ 'failed_tests' ] = [] - repository_status[ 'missing_test_components' ] = [] + if 'missing_test_components' not in repository_status: + repository_status[ 'missing_test_components' ] = [] if not has_test_data: log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' ) # Record the lack of test data. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Update tool_test_results dict's attribute names for consistency. Only flag a changeset revision not to be tested if there no valid tests found in that revision.
by commits-noreply＠bitbucket.org 25 Apr '13

25 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/41d8cdde4729/ Changeset: 41d8cdde4729 User: inithello Date: 2013-04-25 17:13:53 Summary: Update tool_test_results dict's attribute names for consistency. Only flag a changeset revision not to be tested if there no valid tests found in that revision. Affected #: 2 files diff -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 -r 41d8cdde47297746aa82ce4858006bd2632331db lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -130,16 +130,6 @@ "architecture": "x86_64", "system": "Darwin 12.2.0" }, - "test_errors": - [ - { - "test_id": "The test ID, generated by twill", - "tool_id": "The tool ID that was tested", - "tool_version": "The tool version that was tested", - "stderr": "The output of the test, or a more detailed description of what was tested and what the error was." - "traceback": "The traceback, if any." - }, - ] "passed_tests": [ { @@ -147,14 +137,24 @@ "tool_id": "The tool ID that was tested", "tool_version": "The tool version that was tested", }, - ] - "invalid_tests": + ], + "failed_tests": [ { - "tool_id": "The tool ID that does not have functional tests defined.", - "tool_version": "The version of the tool." - "tool_guid": "The guid of the tool." - "reason_test_is_invalid": "A short explanation of what is invalid. + "test_id": "The test ID, generated by twill", + "tool_id": "The tool ID that was tested", + "tool_version": "The tool version that was tested", + "stderr": "The output of the test, or a more detailed description of what was tested and what the error was.", + "traceback": "The traceback, if any." + }, + ], + "missing_test_components": + [ + { + "tool_id": "The tool ID that is missing functional test definitions and/or test data.", + "tool_version": "The version of the tool.", + "tool_guid": "The guid of the tool.", + "missing_components": "The components that are missing for this tool to be considered testable." }, ] } @@ -182,7 +182,7 @@ repository_status = metadata_record.tool_test_results # Clear any old invalid tests for this metadata revision, since this could lead to duplication of invalid test rows, # or tests incorrectly labeled as invalid. - repository_status[ 'invalid_tests' ] = [] + repository_status[ 'missing_test_components' ] = [] if 'test_environment' in repository_status: repository_status[ 'test_environment' ] = get_test_environment( repository_status[ 'test_environment' ] ) else: @@ -204,6 +204,7 @@ continue else: has_test_data = False + testable_revision_found = False # Clone the repository up to the changeset revision we're checking. repo_dir = metadata_record.repository.repo_path( app ) repo = hg.repository( get_configured_ui(), repo_dir ) @@ -254,12 +255,15 @@ failure_reason = '' problem_found = False missing_test_files = [] + has_test_files = False if tool_has_tests and has_test_data: missing_test_files = check_for_missing_test_files( tool_metadata[ 'tests' ], test_data_path ) if missing_test_files: if verbosity >= 2: print "# Tool ID '%s' in changeset revision %s of %s is missing one or more required test files: %s" % \ ( tool_id, changeset_revision, name, ', '.join( missing_test_files ) ) + else: + has_test_files = True if not has_test_data: failure_reason += 'Repository does not have a test-data directory. ' problem_found = True @@ -270,7 +274,7 @@ failure_reason += 'One or more test files are missing for tool %s: %s' % ( tool_id, ', '.join( missing_test_files ) ) problem_found = True test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, - reason_test_is_invalid=failure_reason ) + missing_components=failure_reason ) # The repository_metadata.tool_test_results attribute should always have the following structure: # { # "test_environment": @@ -284,7 +288,15 @@ # "architecture": "x86_64", # "system": "Darwin 12.2.0" # }, - # "test_errors": + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # }, + # ], + # "failed_tests": # [ # { # "test_id": "The test ID, generated by twill", @@ -293,23 +305,14 @@ # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." # "traceback": "The captured traceback." # }, - # ] - # "passed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." - # }, - # ] - # "invalid_tests": + # ], + # "missing_test_components": # [ # { # "tool_id": "The ID of the tool that does not have valid tests.", # "tool_version": "The version of the tool." # "tool_guid": "The guid of the tool." - # "reason_test_is_invalid": "A short explanation of what is invalid." + # "missing_components": "The components that are missing for this tool to be considered testable." # }, # ] # } @@ -318,12 +321,14 @@ # than the list relevant to what it is testing. # Only append this error dict if it hasn't already been added. if problem_found: - if test_errors not in repository_status[ 'invalid_tests' ]: - repository_status[ 'invalid_tests' ].append( test_errors ) + if test_errors not in repository_status[ 'missing_test_components' ]: + repository_status[ 'missing_test_components' ].append( test_errors ) + if tool_has_tests and has_test_files: + testable_revision_found = True # Remove the cloned repository path. This has to be done after the check for required test files, for obvious reasons. if os.path.exists( work_dir ): shutil.rmtree( work_dir ) - if not repository_status[ 'invalid_tests' ]: + if not repository_status[ 'missing_test_components' ]: valid_revisions += 1 if verbosity >= 1: print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) @@ -332,22 +337,27 @@ if verbosity >= 1: print '# Some tools have problematic functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) if verbosity >= 2: - for invalid_test in repository_status[ 'invalid_tests' ]: - if 'reason_test_is_invalid' in invalid_test: - print '# %s' % invalid_test[ 'reason_test_is_invalid' ] + for invalid_test in repository_status[ 'missing_test_components' ]: + if 'missing_components' in invalid_test: + print '# %s' % invalid_test[ 'missing_components' ] if not info_only: # If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform # on which this script was run. - if repository_status[ 'invalid_tests' ]: - # If functional test definitions or test data are missing, set do_not_test = True if and only if: + if repository_status[ 'missing_test_components' ]: + # If functional test definitions or test data are missing, set do_not_test = True if no tool with valid tests has been + # found in this revision, and: # a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision. - # In this case, the revision will never be updated with correct data, and re-testing it would be redundant. - # b) There are one or more downloadable revisions, and the revision being tested is the most recent downloadable revision. - # In this case, if the repository is updated with test data or functional tests, the downloadable changeset revision - # that was tested will be replaced with the new changeset revision, which will be automatically tested. - if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ): + # In this case, the revision will never be updated with the missing components, and re-testing it would be redundant. + # b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable + # revision. In this case, if the repository is updated with test data or functional tests, the downloadable + # changeset revision that was tested will either be replaced with the new changeset revision, or a new downloadable + # changeset revision will be created, either of which will be automatically checked and flagged as appropriate. + # In the install and test script, this behavior is slightly different, since we do want to always run functional + # tests on the most recent downloadable changeset revision. + if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ) and not testable_revision_found: metadata_record.do_not_test = True metadata_record.tools_functionally_correct = False + metadata_record.missing_test_components = True metadata_record.tool_test_results = repository_status metadata_record.time_last_tested = datetime.utcnow() app.sa_session.add( metadata_record ) diff -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 -r 41d8cdde47297746aa82ce4858006bd2632331db test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -174,9 +174,9 @@ def getTestStatus( self, test_identifier ): if test_identifier in self.passed: - tests_passed = self.passed[ test_identifier ] + passed_tests = self.passed[ test_identifier ] del self.passed[ test_identifier ] - return tests_passed + return passed_tests return [] def execute_uninstall_method( repository_dict ): @@ -278,13 +278,13 @@ url_contents = url_handle.read() return from_json_string( url_contents ) -def register_test_result( url, metadata_id, test_results_dict, tests_passed=False ): +def register_test_result( url, metadata_id, test_results_dict, passed_tests=False ): ''' This script should never set do_not_test = True, because the repositories should always be re-tested against the most recent code. ''' params = {} - if tests_passed: + if passed_tests: params[ 'tools_functionally_correct' ] = 'true' params[ 'do_not_test' ] = 'false' else: @@ -599,7 +599,15 @@ # "architecture": "x86_64", # "system": "Darwin 12.2.0" # }, - # "test_errors": + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # }, + # ] + # "failed_tests": # [ # { # "test_id": "The test ID, generated by twill", @@ -609,22 +617,13 @@ # "traceback": "The captured traceback." # }, # ] - # "passed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." - # }, - # ] - # "invalid_tests": + # "missing_test_components": # [ # { # "tool_id": "The tool ID that does not have functional tests defined.", # "tool_version": "The version of the tool." # "tool_guid": "The guid of the tool." - # "reason_test_is_invalid": "A short explanation of what is invalid. + # "missing_components": "A short explanation of what is invalid. # }, # ] # } @@ -635,17 +634,17 @@ test_environment[ 'galaxy_database_version' ] = get_database_version( app ) test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) repository_status[ 'test_environment' ] = test_environment - repository_status[ 'tests_passed' ] = [] - repository_status[ 'test_errors' ] = [] - repository_status[ 'invalid_tests' ] = [] + repository_status[ 'passed_tests' ] = [] + repository_status[ 'failed_tests' ] = [] + repository_status[ 'missing_test_components' ] = [] if not has_test_data: log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' ) # Record the lack of test data. - test_errors = dict( tool_id=None, tool_version=None, tool_guid=None, - reason_test_is_invalid="Repository %s is missing a test-data directory." % name ) - repository_status[ 'invalid_tests' ].append( test_errors ) + failed_tests = dict( tool_id=None, tool_version=None, tool_guid=None, + missing_components="Repository %s is missing a test-data directory." % name ) + repository_status[ 'missing_test_components' ].append( failed_tests ) # Record the status of this repository in the tool shed. - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. execute_uninstall_method( repository_info_dict ) @@ -677,20 +676,20 @@ for plugin in test_plugins: if hasattr( plugin, 'getTestStatus' ): test_identifier = '%s/%s' % ( owner, name ) - tests_passed = plugin.getTestStatus( test_identifier ) + passed_tests = plugin.getTestStatus( test_identifier ) break - repository_status[ 'tests_passed' ] = [] - for test_id in tests_passed: + repository_status[ 'passed_tests' ] = [] + for test_id in passed_tests: tool_id, tool_version = get_tool_info_from_test_id( test_id ) test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) - repository_status[ 'tests_passed' ].append( test_result ) + repository_status[ 'passed_tests' ].append( test_result ) if success: # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and # updates the time_last_tested field to today's date. repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=True ) + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=True ) log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( changeset_revision, name ) ) else: # If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API. @@ -729,13 +728,13 @@ for output_type in [ 'stderr', 'traceback' ]: if output_type in tmp_output: test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) - repository_status[ 'test_errors' ].append( test_status ) + repository_status[ 'failed_tests' ].append( test_status ) # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome # of the tests, and updates tool_test_results with the relevant log data. # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested # field to today's date. repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.' % \ ( changeset_revision, name ) ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Tool shed database migration script to alter the repository_metadata table by dropping the tool_test_errors column and adding columns tool_test_results, missing_test_components.
by commits-noreply＠bitbucket.org 25 Apr '13

25 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f11a2c7a7d32/ Changeset: f11a2c7a7d32 User: greg Date: 2013-04-25 15:36:52 Summary: Tool shed database migration script to alter the repository_metadata table by dropping the tool_test_errors column and adding columns tool_test_results, missing_test_components. Affected #: 17 files diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -42,6 +42,10 @@ tools_functionally_correct = kwd.get( 'tools_functionally_correct', None ) if tools_functionally_correct is not None: clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.string_as_bool( tools_functionally_correct ) ) + # Filter by missing_test_components if received. + missing_test_components = kwd.get( 'missing_test_components', None ) + if missing_test_components is not None: + clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.string_as_bool( missing_test_components ) ) # Filter by do_not_test if received. do_not_test = kwd.get( 'do_not_test', None ) if do_not_test is not None: diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -738,12 +738,7 @@ """ The test framework in ~/test/install_and_test_tool_shed_repositories can be executed on a regularly defined schedule (e.g., via cron) to install appropriate repositories from a tool shed into a Galaxy instance and run defined functional tests for the tools included in the repository. This process affects the values - if these columns in the repository_metadata table: tools_functionally_correct, do_not_test, time_last_tested and tool_test_errors. The tool_test_errors is - slightly mis-named (it should have been named tool_test_results) it will contain a dictionary that includes information about the test environment even if all - tests passed and the tools_functionally_correct column is set to True. - The value of the tool_test_errors column will be a dictionary with the key / value pairs: - "test_environment", {"architecture": "i386", "python_version": "2.5.4", "system": "Darwin 10.8.0"} - "test_errors" [ { "test_id":<some test id>, "stdout":<stdout of running the test>, "stderr":<stderr of running the test>, "traceback":<traceback of running the test>] + if these columns in the repository_metadata table: do_not_test, missing_test_components, time_last_tested, tools_functionally_correct and tool_test_results. """ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -1162,7 +1157,7 @@ trans.model.Repository.table.c.private == False, trans.model.Repository.table.c.deprecated == False, trans.model.Repository.table.c.user_id == user.id ) ): - if not metadata_row.tool_test_errors: + if not metadata_row.tool_test_results: continue # Per the RSS 2.0 specification, all dates in RSS feeds must be formatted as specified in RFC 822 # section 5.1, e.g. Sat, 07 Sep 2002 00:00:01 UT @@ -1171,9 +1166,9 @@ # Generate a citable URL for this repository with owner and changeset revision. repository_citable_url = suc.url_join( tool_shed_url, 'view', user.username, repository.name, metadata_row.changeset_revision ) title = 'Functional test results for changeset revision %s of %s' % ( metadata_row.changeset_revision, repository.name ) - tests_passed = len( metadata_row.tool_test_errors.get( 'tests_passed', [] ) ) - tests_failed = len( metadata_row.tool_test_errors.get( 'test_errors', [] ) ) - invalid_tests = len( metadata_row.tool_test_errors.get( 'invalid_tests', [] ) ) + tests_passed = len( metadata_row.tool_test_results.get( 'tests_passed', [] ) ) + tests_failed = len( metadata_row.tool_test_results.get( 'test_errors', [] ) ) + invalid_tests = len( metadata_row.tool_test_results.get( 'invalid_tests', [] ) ) description = '%d tests passed, %d tests failed, %d tests determined to be invalid.' % ( tests_passed, tests_failed, invalid_tests ) # The guid attribute in an RSS feed's list of items allows a feed reader to choose not to show an item as updated # if the guid is unchanged. For functional test results, the citable URL is sufficiently unique to enable @@ -2674,10 +2669,10 @@ if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ) # TODO: Fix this when the install and test framework is completed. - # if repository_metadata.tool_test_errors: - # tool_test_errors = json.from_json_string( repository_metadata.tool_test_errors ) + # if repository_metadata.tool_test_results: + # tool_test_results = json.from_json_string( repository_metadata.tool_test_results ) # else: - # tool_test_errors = None + # tool_test_results = None metadata = repository_metadata.metadata if metadata: if 'tools' in metadata: @@ -2713,7 +2708,7 @@ else: repository_metadata_id = None metadata = None - #tool_test_errors = None + #tool_test_results = None is_malicious = suc.changeset_is_malicious( trans, repository_id, repository.tip( trans.app ) ) changeset_revision_select_field = grids_util.build_changeset_revision_select_field( trans, repository, @@ -2737,7 +2732,7 @@ tool=tool, tool_metadata_dict=tool_metadata_dict, tool_lineage=tool_lineage, - #tool_test_errors=tool_test_errors, + #tool_test_results=tool_test_results, changeset_revision=changeset_revision, revision_label=revision_label, changeset_revision_select_field=changeset_revision_select_field, diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -28,7 +28,7 @@ repositories_without_reviews_grid = repository_review_grids.RepositoriesWithoutReviewsGrid() repository_reviews_by_user_grid = repository_review_grids.RepositoryReviewsByUserGrid() reviewed_repositories_i_own_grid = repository_review_grids.ReviewedRepositoriesIOwnGrid() - repositories_with_invalid_tests_grid = repository_review_grids.RepositoriesWithInvalidTestsGrid() + repositories_with_no_tool_tests_grid = repository_review_grids.RepositoriesWithNoToolTestsGrid() @web.expose @web.require_login( "approve repository review" ) @@ -417,7 +417,10 @@ @web.expose @web.require_login( "manage repositories with invalid tests" ) def manage_repositories_with_invalid_tests( self, trans, **kwd ): - """Display a list of repositories that contain tools, have not yet been reviewed, and have invalid functional tests.""" + """ + Display a list of repositories that contain tools, have not yet been reviewed, and have invalid functional tests. Tests are defined as + invalid if they are missing from the tool config or if defined test data is not included in the repository. + """ if 'operation' in kwd: operation = kwd['operation'].lower() if operation == "inspect repository revisions": @@ -428,10 +431,10 @@ return trans.response.send_redirect( web.url_for( controller='repository_review', action='view_or_manage_repository', **kwd ) ) - message = 'These repositories contain tools with invalid functional tests (they have not yet been reviewed). ' + message = 'These repositories contain tools with missing functional tests or test data. ' kwd[ 'message' ] = message kwd[ 'status' ] = 'warning' - return self.repositories_with_invalid_tests_grid( trans, **kwd ) + return self.repositories_with_no_tool_tests_grid( trans, **kwd ) @web.expose @web.require_login( "manage repositories with reviews" ) diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -1,28 +1,27 @@ -""" -Galaxy Tool Shed data model classes - -Naming: try to use class names that have a distinct plural form so that -the relationship cardinalities are obvious (e.g. prefer Dataset to Data) -""" -import os.path, os, errno, sys, codecs, operator, logging, tarfile, mimetypes, ConfigParser +import logging +import operator +import os from galaxy import util from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash -from galaxy.web.form_builder import * from galaxy.model.item_attrs import APIItem from galaxy import eggs -eggs.require('mercurial') -from mercurial import hg, ui +eggs.require( 'mercurial' ) +from mercurial import hg +from mercurial import ui log = logging.getLogger( __name__ ) + class APIKeys( object ): pass + class User( object, APIItem ): api_collection_visible_keys = ( 'id', 'email' ) api_element_visible_keys = ( 'id', 'email', 'username' ) + def __init__( self, email=None, password=None ): self.email = email self.password = password @@ -31,6 +30,7 @@ self.purged = False self.username = None self.new_repo_alert = False + def all_roles( self ): roles = [ ura.role for ura in self.roles ] for group in [ uga.group for uga in self.groups ]: @@ -38,28 +38,37 @@ if role not in roles: roles.append( role ) return roles + def set_password_cleartext( self, cleartext ): """Set 'self.password' to the digest of 'cleartext'.""" self.password = new_secure_hash( text_type=cleartext ) + def check_password( self, cleartext ): """Check if 'cleartext' matches 'self.password' when hashed.""" return self.password == new_secure_hash( text_type=cleartext ) + def get_disk_usage( self, nice_size=False ): return 0 + def set_disk_usage( self, bytes ): pass + total_disk_usage = property( get_disk_usage, set_disk_usage ) + @property def nice_total_disk_usage( self ): return 0 + class Group( object, APIItem ): api_collection_visible_keys = ( 'id', 'name' ) api_element_visible_keys = ( 'id', 'name' ) + def __init__( self, name = None ): self.name = name self.deleted = False + class Role( object, APIItem ): api_collection_visible_keys = ( 'id', 'name' ) api_element_visible_keys = ( 'id', 'name', 'description', 'type' ) @@ -71,28 +80,34 @@ ADMIN = 'admin', SHARING = 'sharing' ) + def __init__( self, name="", description="", type="system", deleted=False ): self.name = name self.description = description self.type = type self.deleted = deleted + class UserGroupAssociation( object ): def __init__( self, user, group ): self.user = user self.group = group + class UserRoleAssociation( object ): def __init__( self, user, role ): self.user = user self.role = role + class GroupRoleAssociation( object ): def __init__( self, group, role ): self.group = group self.role = role + class GalaxySession( object ): + def __init__( self, id=None, user=None, @@ -113,6 +128,7 @@ self.is_valid = is_valid self.prev_session_id = prev_session_id + class Repository( object, APIItem ): api_collection_visible_keys = ( 'id', 'name', 'description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' ) api_element_visible_keys = ( 'id', 'name', 'description', 'long_description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' ) @@ -121,6 +137,7 @@ MARKED_FOR_REMOVAL = 'r', MARKED_FOR_ADDITION = 'a', NOT_TRACKED = '?' ) + def __init__( self, id=None, name=None, description=None, long_description=None, user_id=None, private=False, deleted=None, email_alerts=None, times_downloaded=0, deprecated=False ): self.id = id @@ -133,8 +150,10 @@ self.email_alerts = email_alerts self.times_downloaded = times_downloaded self.deprecated = deprecated + def as_dict( self, value_mapper=None ): return self.get_api_value( view='element', value_mapper=value_mapper ) + def get_api_value( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} @@ -153,22 +172,28 @@ if 'user_id' in rval: rval[ 'owner' ] = self.user.username return rval + def repo_path( self, app ): return app.hgweb_config_manager.get_entry( os.path.join( "repos", self.user.username, self.name ) ) + def revision( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) tip_ctx = repo.changectx( repo.changelog.tip() ) return "%s:%s" % ( str( tip_ctx.rev() ), str( repo.changectx( repo.changelog.tip() ) ) ) + def tip( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) return str( repo.changectx( repo.changelog.tip() ) ) + def is_new( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) tip_ctx = repo.changectx( repo.changelog.tip() ) return tip_ctx.rev() < 0 + def allow_push( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) return repo.ui.config( 'web', 'allow_push' ) + def set_allow_push( self, app, usernames, remove_auth='' ): allow_push = util.listify( self.allow_push( app ) ) if remove_auth: @@ -190,15 +215,17 @@ fp.write( line ) fp.close() + class RepositoryMetadata( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', - 'do_not_test', 'time_last_tested', 'tool_test_errors', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', + 'do_not_test', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) + def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, - tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None, has_repository_dependencies=False, - includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): + missing_test_components=None, tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_results=None, + has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): self.id = id self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -206,15 +233,17 @@ self.tool_versions = tool_versions or dict() self.malicious = malicious self.downloadable = downloadable + self.missing_test_components = missing_test_components self.tools_functionally_correct = tools_functionally_correct self.do_not_test = do_not_test self.time_last_tested = time_last_tested - self.tool_test_errors = tool_test_errors + self.tool_test_results = tool_test_results self.has_repository_dependencies = has_repository_dependencies self.includes_datatypes = includes_datatypes self.includes_tools = includes_tools self.includes_tool_dependencies = includes_tool_dependencies self.includes_workflows = includes_workflows + @property def includes_tools_for_display_in_tool_panel( self ): if self.metadata: @@ -223,8 +252,10 @@ if tool_dict.get( 'add_to_tool_panel', True ): return True return False + def as_dict( self, value_mapper=None ): return self.get_api_value( view='element', value_mapper=value_mapper ) + def get_api_value( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} @@ -242,10 +273,12 @@ rval[ key ] = None return rval + class RepositoryReview( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) approved_states = Bunch( NO='no', YES='yes' ) + def __init__( self, repository_id=None, changeset_revision=None, user_id=None, rating=None, deleted=False ): self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -257,6 +290,7 @@ api_collection_visible_keys = ( 'id', 'repository_review_id', 'component_id', 'private', 'approved', 'rating', 'deleted' ) api_element_visible_keys = ( 'id', 'repository_review_id', 'component_id', 'private', 'approved', 'rating', 'deleted' ) approved_states = Bunch( NO='no', YES='yes', NA='not_applicable' ) + def __init__( self, repository_review_id=None, component_id=None, comment=None, private=False, approved=False, rating=None, deleted=False ): self.repository_review_id = repository_review_id self.component_id = component_id @@ -266,49 +300,65 @@ self.rating = rating self.deleted = deleted + class Component( object ): + def __init__( self, name=None, description=None ): self.name = name self.description = description + class ItemRatingAssociation( object ): + def __init__( self, id=None, user=None, item=None, rating=0, comment='' ): self.id = id self.user = user self.item = item self.rating = rating self.comment = comment + def set_item( self, item ): """ Set association's item. """ pass + class RepositoryRatingAssociation( ItemRatingAssociation ): + def set_item( self, repository ): self.repository = repository + class Category( object, APIItem ): api_collection_visible_keys = ( 'id', 'name', 'description', 'deleted' ) api_element_visible_keys = ( 'id', 'name', 'description', 'deleted' ) + def __init__( self, name=None, description=None, deleted=False ): self.name = name self.description = description self.deleted = deleted + class RepositoryCategoryAssociation( object ): + def __init__( self, repository=None, category=None ): self.repository = repository self.category = category + class Tag( object ): + def __init__( self, id=None, type=None, parent_id=None, name=None ): self.id = id self.type = type self.parent_id = parent_id self.name = name + def __str__ ( self ): return "Tag(id=%s, type=%i, parent_id=%s, name=%s)" % ( self.id, self.type, self.parent_id, self.name ) + class ItemTagAssociation( object ): + def __init__( self, id=None, user=None, item_id=None, tag_id=None, user_tname=None, value=None ): self.id = id self.user = user @@ -318,7 +368,9 @@ self.value = None self.user_value = None + class Workflow( object ): + def __init__( self ): self.user = None self.name = None @@ -326,7 +378,9 @@ self.has_errors = None self.steps = [] + class WorkflowStep( object ): + def __init__( self ): self.id = None self.type = None @@ -336,17 +390,20 @@ self.tool_errors = None self.position = None self.input_connections = [] - #self.output_connections = [] self.config = None + class WorkflowStepConnection( object ): + def __init__( self ): self.output_step = None self.output_name = None self.input_step = None self.input_name = None + ## ---- Utility methods ------------------------------------------------------- + def sort_by_attr( seq, attr ): """ Sort the sequence of objects by object's attribute @@ -362,6 +419,7 @@ intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) intermed.sort() return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) + def directory_hash_id( id ): s = str( id ) l = len( s ) diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/model/mapping.py --- a/lib/galaxy/webapps/tool_shed/model/mapping.py +++ b/lib/galaxy/webapps/tool_shed/model/mapping.py @@ -134,7 +134,8 @@ Column( "tools_functionally_correct", Boolean, default=False, index=True ), Column( "do_not_test", Boolean, default=False, index=True ), Column( "time_last_tested", DateTime, default=None, nullable=True ), - Column( "tool_test_errors", JSONType, nullable=True ), + Column( "missing_test_components", Boolean, default=False, index=True ), + Column( "tool_test_results", JSONType, nullable=True ), Column( "has_repository_dependencies", Boolean, default=False, index=True ), Column( "includes_datatypes", Boolean, default=False, index=True ), Column( "includes_tools", Boolean, default=False, index=True ), diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/model/migrate/versions/0018_add_repository_metadata_flag_columns.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0018_add_repository_metadata_flag_columns.py @@ -0,0 +1,90 @@ +""" +Migration script to alter the repository_metadata table by dropping the tool_test_errors column and adding columns +tool_test_results, missing_test_components. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + + try: + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + except NoSuchTableError: + RepositoryMetadata_table = None + log.debug( "Failed loading table repository_metadata." ) + + if RepositoryMetadata_table: + # Drop the tool_test_errors column from the repository_metadata table as it is poorly named. It will be replaced with the new + # tool_test_results column. + try: + col = RepositoryMetadata_table.c.tool_test_errors + col.drop() + except Exception, e: + log.debug( "Dropping column 'tool_test_errors' from repository_metadata table failed: %s" % ( str( e ) ) ) + + # Create the tool_test_results column to replace the ill-named tool_test_errors column just dropped above. + c = Column( "tool_test_results", JSONType, nullable=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.tool_test_results + except Exception, e: + print "Adding tool_test_results column to the repository_metadata table failed: %s" % str( e ) + + # Create the missing_test_components column. + c = Column( "missing_test_components", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.missing_test_components + db_session.execute( "UPDATE repository_metadata SET missing_test_components=%s" % default_false ) + except Exception, e: + print "Adding missing_test_components column to the repository_metadata table failed: %s" % str( e ) + +def downgrade(): + metadata.reflect() + # Drop missing_test_components and tool_test_results from the repository_metadata table and add tool_test_errors to the repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + + # Drop the missing_test_components column. + try: + RepositoryMetadata_table.c.missing_test_components.drop() + except Exception, e: + print "Dropping column missing_test_components from the repository_metadata table failed: %s" % str( e ) + + # Drop the tool_test_results column. + try: + RepositoryMetadata_table.c.tool_test_results.drop() + except Exception, e: + print "Dropping column tool_test_results from the repository_metadata table failed: %s" % str( e ) + + # Create the tool_test_errors column. + c = Column( "tool_test_errors", JSONType, nullable=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.tool_test_errors + except Exception, e: + print "Adding tool_test_errors column to the repository_metadata table failed: %s" % str( e ) diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/tool_shed/grids/repository_review_grids.py --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -395,12 +395,12 @@ .outerjoin( ( model.Component.table, model.Component.table.c.id == model.ComponentReview.table.c.component_id ) ) -class RepositoriesWithInvalidTestsGrid( RepositoriesWithoutReviewsGrid ): +class RepositoriesWithNoToolTestsGrid( RepositoriesWithoutReviewsGrid ): # Repositories that are ready for human review are those that either: # 1) Have no tools # 2) Have tools that have been proven to be functionally correct within Galaxy. # This grid filters out repositories that have been marked as either deprecated or deleted. - title = "Repositories that contain tools with invalid functional tests" + title = "Repositories that contain tools with no tests or test data" columns = [ RepositoriesWithoutReviewsGrid.NameColumn( "Repository name", key="name", @@ -428,8 +428,7 @@ def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False, - model.Repository.reviews == None ) ) \ + model.Repository.table.c.deprecated == False ) ) \ .join( model.RepositoryMetadata.table ) \ .filter( and_( model.RepositoryMetadata.table.c.downloadable == True, model.RepositoryMetadata.table.c.includes_tools == True, diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py --- a/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py +++ b/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py @@ -21,14 +21,14 @@ for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]: """ This example script will properly handle updating the value of one or more of the following RepositoryMetadata attributes: - tools_functionally_correct, do_not_test, tool_test_errors + tools_functionally_correct, do_not_test, tool_test_results """ if key in [ 'tools_functionally_correct', 'do_not_test' ]: if str( value ).lower() in [ 'true', 'yes', 'on' ]: new_value = True else: new_value = False - elif key in [ 'tool_test_errors' ]: + elif key in [ 'tool_test_results' ]: new_value = from_json_string( value ) else: new_value = str( value ) diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -117,7 +117,7 @@ and test repositories script to process. If the tested changeset revision does not have a test-data directory, this script will also mark the revision not to be tested. - If any error is encountered, the script will update the repository_metadata.tool_test_errors attribute following this structure: + If any error is encountered, the script will update the repository_metadata.tool_test_results attribute following this structure: { "test_environment": { @@ -178,8 +178,8 @@ for metadata_record in metadata_records_to_check: # Initialize the repository_status dict with the test environment, but leave the test_errors empty. repository_status = {} - if metadata_record.tool_test_errors: - repository_status = metadata_record.tool_test_errors + if metadata_record.tool_test_results: + repository_status = metadata_record.tool_test_results # Clear any old invalid tests for this metadata revision, since this could lead to duplication of invalid test rows, # or tests incorrectly labeled as invalid. repository_status[ 'invalid_tests' ] = [] @@ -271,7 +271,7 @@ problem_found = True test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, reason_test_is_invalid=failure_reason ) - # The repository_metadata.tool_test_errors attribute should always have the following structure: + # The repository_metadata.tool_test_results attribute should always have the following structure: # { # "test_environment": # { @@ -348,7 +348,7 @@ if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ): metadata_record.do_not_test = True metadata_record.tools_functionally_correct = False - metadata_record.tool_test_errors = repository_status + metadata_record.tool_test_results = repository_status metadata_record.time_last_tested = datetime.utcnow() app.sa_session.add( metadata_record ) app.sa_session.flush() diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -277,9 +277,10 @@ includes_workflows=includes_workflows ) # Always set the default values for the following columns. When resetting all metadata on a repository, this will reset the values. repository_metadata.tools_functionally_correct = False + repository_metadata.missing_test_components = False repository_metadata.do_not_test = False repository_metadata.time_last_tested = None - repository_metadata.tool_test_errors = None + repository_metadata.tool_test_results = None trans.sa_session.add( repository_metadata ) trans.sa_session.flush() return repository_metadata @@ -1722,7 +1723,8 @@ repository_metadata.do_not_test = False repository_metadata.time_last_tested = None repository_metadata.tools_functionally_correct = False - repository_metadata.tool_test_errors = None + repository_metadata.missing_test_components = False + repository_metadata.tool_test_results = None trans.sa_session.add( repository_metadata ) trans.sa_session.flush() else: diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/admin/index.mako --- a/templates/webapps/tool_shed/admin/index.mako +++ b/templates/webapps/tool_shed/admin/index.mako @@ -76,9 +76,6 @@ <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_ready_for_review' )}">Repositories ready for review</a></div><div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_with_invalid_tests' )}">Repositories with invalid tests</a> - </div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_without_reviews' )}">All repositories with no reviews</a></div> %if trans.user.repository_reviews: @@ -94,6 +91,17 @@ </div></div></div> + <div class="toolSectionPad"></div> + <div class="toolSectionTitle"> + Reviewing Repositories With Tools + </div> + <div class="toolSectionBody"> + <div class="toolSectionBg"> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_with_invalid_tests' )}">Repositories missing tests or data</a> + </div> + </div> + </div> %endif <div class="toolSectionPad"></div><div class="toolSectionTitle"> diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/index.mako --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -133,9 +133,6 @@ <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_ready_for_review' )}">Repositories ready for review</a></div><div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_with_invalid_tests' )}">Repositories with invalid tests</a> - </div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_without_reviews' )}">All repositories with no reviews</a></div> %if trans.user.repository_reviews: @@ -151,6 +148,17 @@ </div></div></div> + <div class="toolSectionPad"></div> + <div class="toolSectionTitle"> + Reviewing Repositories With Tools + </div> + <div class="toolSectionBody"> + <div class="toolSectionBg"> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_with_invalid_tests' )}">Repositories missing tests or data</a> + </div> + </div> + </div> %endif %else: <div class="toolSectionPad"></div> diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako --- a/templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako +++ b/templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako @@ -51,16 +51,16 @@ can_review_repository = has_metadata and not is_deprecated and trans.app.security_agent.user_can_review_repositories( trans.user ) can_upload = can_push can_view_change_log = trans.webapp.name == 'tool_shed' and not is_new - if repository_metadata.tool_test_errors: - # The tool_test_errors is mis-named (it should have been named tool_test_results) it will contain a dictionary that includes information - # about the test environment even if all tests passed and the repository_metadata.tools_functionally_correct column is set to True. - tool_test_errors = repository_metadata.tool_test_errors - test_environment_dict = tool_test_errors.get( 'test_environment', None ) - invalid_tests = tool_test_errors.get( 'invalid_tests', [] ) - test_errors = tool_test_errors.get( 'test_errors', [] ) - tests_passed = tool_test_errors.get( 'tests_passed', [] ) + if repository_metadata.tool_test_results: + # The tool_test_results will contain a dictionary that includes information about the test environment even if all tests passed and the + # repository_metadata.tools_functionally_correct column is set to True. + tool_test_results = repository_metadata.tool_test_results + test_environment_dict = tool_test_results.get( 'test_environment', None ) + invalid_tests = tool_test_results.get( 'invalid_tests', [] ) + test_errors = tool_test_results.get( 'test_errors', [] ) + tests_passed = tool_test_results.get( 'tests_passed', [] ) else: - tool_test_errors = None + tool_test_results = None test_environment_dict = {} invalid_tests = [] test_errors = [] @@ -144,7 +144,7 @@ <b>Repository name:</b><br/> ${repository.name} %endif -%if invalid_tests or tool_test_errors or tests_passed: +%if invalid_tests or tool_test_results or tests_passed: <p/><div class="toolForm"><div class="toolFormTitle">Tool functional test results</div> diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -27,7 +27,7 @@ can_upload = can_push can_view_change_log = not is_new if repository_metadata: - if repository_metadata.includes_tools and repository_metadata.tool_test_errors is not None: + if repository_metadata.includes_tools and repository_metadata.tool_test_results is not None: can_display_tool_functional_test_results = True else: can_display_tool_functional_test_results = False diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/repository/view_repository.mako --- a/templates/webapps/tool_shed/repository/view_repository.mako +++ b/templates/webapps/tool_shed/repository/view_repository.mako @@ -26,7 +26,7 @@ changeset_revision_is_repository_tip = changeset_revision == repository.tip( trans.app ) if repository_metadata: - if repository_metadata.includes_tools and repository_metadata.tool_test_errors is not None: + if repository_metadata.includes_tools and repository_metadata.tool_test_results is not None: can_display_tool_functional_test_results = True else: can_display_tool_functional_test_results = False diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/repository/view_tool_metadata.mako --- a/templates/webapps/tool_shed/repository/view_tool_metadata.mako +++ b/templates/webapps/tool_shed/repository/view_tool_metadata.mako @@ -23,7 +23,7 @@ can_upload = can_push can_view_change_log = trans.webapp.name == 'tool_shed' and not is_new # TODO: fix the following when the install and test buildbot is functional. - #can_view_tool_test_errors = tool_test_errors is not None + #can_view_tool_test_results = tool_test_results is not None if can_push: browse_label = 'Browse or delete repository tip files' diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -265,13 +265,13 @@ tool_id = parts[ -2 ] return tool_id, tool_version -def get_tool_test_errors_from_api( tool_shed_url, metadata_revision_id ): +def get_tool_test_results_from_api( tool_shed_url, metadata_revision_id ): api_path = [ 'api', 'repository_revisions', metadata_revision_id ] api_url = get_api_url( base=tool_shed_url, parts=api_path ) repository_metadata = json_from_url( api_url ) - if repository_metadata[ 'tool_test_errors' ] is None: + if repository_metadata[ 'tool_test_results' ] is None: return {} - return repository_metadata[ 'tool_test_errors' ] + return repository_metadata[ 'tool_test_results' ] def json_from_url( url ): url_handle = urllib.urlopen( url ) @@ -290,7 +290,7 @@ else: params[ 'tools_functionally_correct' ] = 'false' params[ 'do_not_test' ] = 'false' - params[ 'tool_test_errors' ] = test_results_dict + params[ 'tool_test_results' ] = test_results_dict if '-info_only' in sys.argv: return {} else: @@ -583,7 +583,7 @@ log.debug( 'Installation of %s succeeded, running all defined functional tests.' % name ) # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, - # and the tools_functionally_correct flag to False, as well as updating tool_test_errors. + # and the tools_functionally_correct flag to False, as well as updating tool_test_results. file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( dict() ) ) has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) # The repository_status dict should always have the following structure: @@ -628,7 +628,7 @@ # }, # ] # } - repository_status = get_tool_test_errors_from_api( galaxy_tool_shed_url, metadata_revision_id ) + repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) if 'test_environment' not in repository_status: repository_status[ 'test_environment' ] = {} test_environment = get_test_environment( repository_status[ 'test_environment' ] ) @@ -731,7 +731,7 @@ test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) repository_status[ 'test_errors' ].append( test_status ) # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome - # of the tests, and updates tool_test_errors with the relevant log data. + # of the tests, and updates tool_test_results with the relevant log data. # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested # field to today's date. repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 25 Apr '13

25 Apr '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/33bec3cae5b7/ Changeset: 33bec3cae5b7 User: dannon Date: 2013-04-25 14:13:16 Summary: Organize multi.py imports. Affected #: 1 file diff -r 75c5ab4e924192528a6d250d61109c05d16f0118 -r 33bec3cae5b7731916349977918c847f7b255267 lib/galaxy/jobs/splitters/multi.py --- a/lib/galaxy/jobs/splitters/multi.py +++ b/lib/galaxy/jobs/splitters/multi.py @@ -1,5 +1,8 @@ -import os, logging, shutil +import os +import logging +import shutil import inspect + from galaxy import model, util https://bitbucket.org/galaxy/galaxy-central/commits/a599aba4d18c/ Changeset: a599aba4d18c User: dannon Date: 2013-04-25 14:19:33 Summary: Patch from Peter Cock to ensure stdout/stderr are separated in dataset info. Affected #: 1 file diff -r 33bec3cae5b7731916349977918c847f7b255267 -r a599aba4d18c634c51d40c68c64148e45f4e083b lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -924,7 +924,13 @@ for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: #need to update all associated output hdas, i.e. history was shared with job running dataset.blurb = 'done' dataset.peek = 'no peek' - dataset.info = ( dataset.info or '' ) + context['stdout'] + context['stderr'] + dataset.info = (dataset.info or '') + if context['stdout'].strip(): + #Ensure white space between entries + dataset.info = dataset.info.rstrip() + "\n" + context['stdout'].strip() + if context['stderr'].strip(): + #Ensure white space between entries + dataset.info = dataset.info.rstrip() + "\n" + context['stderr'].strip() dataset.tool_version = self.version_string dataset.set_size() if 'uuid' in context: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 25 Apr '13

25 Apr '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1191f246a22b/ Changeset: 1191f246a22b User: dannon Date: 2013-04-25 14:09:52 Summary: Add shutil to binary.py as pointed out by Peter. Affected #: 1 file diff -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 -r 1191f246a22b122bfdc27fe6f0488d1b0d3f0dbb lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -2,18 +2,26 @@ Binary classes """ -import data, logging, binascii -from galaxy.datatypes.metadata import MetadataElement -from galaxy.datatypes import metadata -from galaxy.datatypes.sniff import * +import binascii +import data +import gzip +import logging +import os +import shutil +import struct +import subprocess +import tempfile +import zipfile + +from urllib import urlencode, quote_plus from galaxy import eggs import pkg_resources pkg_resources.require( "bx-python" ) from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE -from urllib import urlencode, quote_plus -import zipfile, gzip -import os, subprocess, tempfile -import struct + +from galaxy.datatypes.metadata import MetadataElement +from galaxy.datatypes import metadata +from galaxy.datatypes.sniff import * log = logging.getLogger(__name__) https://bitbucket.org/galaxy/galaxy-central/commits/75c5ab4e9241/ Changeset: 75c5ab4e9241 User: dannon Date: 2013-04-25 14:10:46 Summary: Add missing shutil import to data.py as pointed out by Peter. Affected #: 1 file diff -r 1191f246a22b122bfdc27fe6f0488d1b0d3f0dbb -r 75c5ab4e924192528a6d250d61109c05d16f0118 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -2,6 +2,7 @@ import metadata import mimetypes import os +import shutil import sys import tempfile import zipfile Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: HDA model: add UsesAnnotations mixin; HDA API: add update method and allow name, deleted, visible, genome_build, dbkey, info, and annotation to be updated; History & HDA API: don't error on allowed but uneditable keys; Browser tests: test hda api
by commits-noreply＠bitbucket.org 24 Apr '13

24 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8b9ca63f9128/ Changeset: 8b9ca63f9128 User: carlfeberhard Date: 2013-04-24 22:24:41 Summary: HDA model: add UsesAnnotations mixin; HDA API: add update method and allow name, deleted, visible, genome_build, dbkey, info, and annotation to be updated; History & HDA API: don't error on allowed but uneditable keys; Browser tests: test hda api Affected #: 9 files diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -727,6 +727,7 @@ """ # precondition: keys are proper, values are parsed and validated changed = {} + # unknown keys are ignored here for key in [ k for k in new_data.keys() if k in self.api_element_visible_keys ]: new_val = new_data[ key ] old_val = self.__getattribute__( key ) @@ -1428,7 +1429,11 @@ return msg -class HistoryDatasetAssociation( DatasetInstance ): +class HistoryDatasetAssociation( DatasetInstance, UsesAnnotations ): + """ + Resource class that creates a relation between a dataset and a user history. + """ + def __init__( self, hid = None, history = None, @@ -1436,6 +1441,9 @@ copied_from_library_dataset_dataset_association = None, sa_session = None, **kwd ): + """ + Create a a new HDA and associate it with the given history. + """ # FIXME: sa_session is must be passed to DataSetInstance if the create_dataset # parameter is True so that the new object can be flushed. Is there a better way? DatasetInstance.__init__( self, sa_session=sa_session, **kwd ) @@ -1444,7 +1452,11 @@ self.history = history self.copied_from_history_dataset_association = copied_from_history_dataset_association self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association + def copy( self, copy_children = False, parent_id = None ): + """ + Create a copy of this HDA. + """ hda = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, @@ -1471,13 +1483,20 @@ hda.set_peek() object_session( self ).flush() return hda - def to_library_dataset_dataset_association( self, trans, target_folder, replace_dataset=None, parent_id=None, user=None, roles=[], ldda_message='' ): + + def to_library_dataset_dataset_association( self, trans, target_folder, + replace_dataset=None, parent_id=None, user=None, roles=[], ldda_message='' ): + """ + Copy this HDA to a library optionally replacing an existing LDDA. + """ if replace_dataset: - # The replace_dataset param ( when not None ) refers to a LibraryDataset that is being replaced with a new version. + # The replace_dataset param ( when not None ) refers to a LibraryDataset that + # is being replaced with a new version. library_dataset = replace_dataset else: - # If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new - # LibraryDataset, and the current user's DefaultUserPermissions will be applied to the associated Dataset. + # If replace_dataset is None, the Library level permissions will be taken from the folder and + # applied to the new LibraryDataset, and the current user's DefaultUserPermissions will be applied + # to the associated Dataset. library_dataset = LibraryDataset( folder=target_folder, name=self.name, info=self.info ) object_session( self ).add( library_dataset ) object_session( self ).flush() @@ -1502,7 +1521,8 @@ object_session( self ).flush() # If roles were selected on the upload form, restrict access to the Dataset to those roles for role in roles: - dp = trans.model.DatasetPermissions( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, ldda.dataset, role ) + dp = trans.model.DatasetPermissions( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, + ldda.dataset, role ) trans.sa_session.add( dp ) trans.sa_session.flush() # Must set metadata after ldda flushed, as MetadataFiles require ldda.id @@ -1527,30 +1547,47 @@ ldda.set_peek() object_session( self ).flush() return ldda + def clear_associated_files( self, metadata_safe = False, purge = False ): + """ + """ # metadata_safe = True means to only clear when assoc.metadata_safe == False for assoc in self.implicitly_converted_datasets: if not assoc.deleted and ( not metadata_safe or not assoc.metadata_safe ): assoc.clear( purge = purge ) for assoc in self.implicitly_converted_parent_datasets: assoc.clear( purge = purge, delete_dataset = False ) + def get_display_name( self ): - ## Name can be either a string or a unicode object. If string, convert to unicode object assuming 'utf-8' format. + """ + Return the name of this HDA in either ascii or utf-8 encoding. + """ + # Name can be either a string or a unicode object. + # If string, convert to unicode object assuming 'utf-8' format. hda_name = self.name if isinstance(hda_name, str): hda_name = unicode(hda_name, 'utf-8') return hda_name + def get_access_roles( self, trans ): + """ + Return The access roles associated with this HDA's dataset. + """ return self.dataset.get_access_roles( trans ) + def quota_amount( self, user ): """ - If the user has multiple instances of this dataset, it will not affect their disk usage statistic. + Return the disk space used for this HDA relevant to user quotas. + + If the user has multiple instances of this dataset, it will not affect their + disk usage statistic. """ rval = 0 # Anon users are handled just by their single history size. if not user: return rval - # Gets an HDA and its children's disk usage, if the user does not already have an association of the same dataset + # Gets an HDA and its children's disk usage, if the user does not already + # have an association of the same dataset if not self.dataset.library_associations and not self.purged and not self.dataset.purged: for hda in self.dataset.history_associations: if hda.id == self.id: @@ -1562,7 +1599,11 @@ for child in self.children: rval += child.get_disk_usage( user ) return rval + def get_api_value( self, view='collection' ): + """ + Return attributes of this HDA that are exposed using the API. + """ # Since this class is a proxy to rather complex attributes we want to # display in other objects, we can't use the simpler method used by # other model classes. @@ -1598,6 +1639,33 @@ rval['metadata_' + name] = val return rval + def set_from_dict( self, new_data ): + #AKA: set_api_value + """ + Set object attributes to the values in dictionary new_data limiting + to only the following keys: name, deleted, visible, genome_build, + info, and blurb. + + Returns a dictionary of the keys, values that have been changed. + """ + # precondition: keys are proper, values are parsed and validated + #NOTE!: does not handle metadata + editable_keys = ( 'name', 'deleted', 'visible', 'dbkey', 'info', 'blurb' ) + + changed = {} + # unknown keys are ignored here + for key in [ k for k in new_data.keys() if k in editable_keys ]: + new_val = new_data[ key ] + old_val = self.__getattribute__( key ) + if new_val == old_val: + continue + + self.__setattr__( key, new_val ) + changed[ key ] = new_val + + return changed + + class HistoryDatasetAssociationDisplayAtAuthorization( object ): def __init__( self, hda=None, user=None, site=None ): self.history_dataset_association = hda diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -427,10 +427,15 @@ class UsesHistoryDatasetAssociationMixin: - """ Mixin for controllers that use HistoryDatasetAssociation objects. """ + """ + Mixin for controllers that use HistoryDatasetAssociation objects. + """ def get_dataset( self, trans, dataset_id, check_ownership=True, check_accessible=False, check_state=True ): - """ Get an HDA object by id. """ + """ + Get an HDA object by id performing security checks using + the current transaction. + """ # DEPRECATION: We still support unencoded ids for backward compatibility try: # encoded id? @@ -466,7 +471,10 @@ def get_history_dataset_association( self, trans, history, dataset_id, check_ownership=True, check_accessible=False, check_state=False ): - """Get a HistoryDatasetAssociation from the database by id, verifying ownership.""" + """ + Get a HistoryDatasetAssociation from the database by id, verifying ownership. + """ + #TODO: duplicate of above? alias to above (or vis-versa) self.security_check( trans, history, check_ownership=check_ownership, check_accessible=check_accessible ) hda = self.get_object( trans, dataset_id, 'HistoryDatasetAssociation', check_ownership=False, check_accessible=False, deleted=False ) @@ -479,8 +487,9 @@ return hda def get_data( self, dataset, preview=True ): - """ Gets a dataset's data. """ - + """ + Gets a dataset's data. + """ # Get data from file, truncating if necessary. truncated = False dataset_data = None @@ -610,6 +619,27 @@ return display_apps + def set_hda_from_dict( self, trans, hda, new_data ): + """ + Changes HDA data using the given dictionary new_data. + """ + # precondition: access of the hda has already been checked + + # send what we can down into the model + changed = hda.set_from_dict( new_data ) + # the rest (often involving the trans) - do here + if 'annotation' in new_data.keys() and trans.get_user(): + hda.add_item_annotation( trans.sa_session, trans.get_user(), hda, new_data[ 'annotation' ] ) + changed[ 'annotation' ] = new_data[ 'annotation' ] + # tags + # sharing/permissions? + # purged + + if changed.keys(): + trans.sa_session.flush() + + return changed + class UsesLibraryMixin: diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -2,15 +2,10 @@ API operations on a history. """ -import pkg_resources -pkg_resources.require("Paste") -from paste.httpexceptions import HTTPBadRequest - from galaxy import web, util from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin from galaxy.web import url_for from galaxy.model.orm import desc -from galaxy.util.bunch import Bunch import logging log = logging.getLogger( __name__ ) @@ -197,28 +192,39 @@ # - protection against bad data form/type # - protection against malicious data content # all other conversions and processing (such as permissions, etc.) should happen down the line + + # keys listed here don't error when attempting to set, but fail silently + # this allows PUT'ing an entire model back to the server without attribute errors on uneditable attrs + valid_but_uneditable_keys = ( + 'id', 'model_class', 'nice_size', 'contents_url', 'purged', 'tags', + 'state', 'state_details', 'state_ids' + ) + + validated_payload = {} for key, val in payload.items(): # TODO: lots of boilerplate here, but overhead on abstraction is equally onerous if key == 'name': if not ( isinstance( val, str ) or isinstance( val, unicode ) ): raise ValueError( 'name must be a string or unicode: %s' %( str( type( val ) ) ) ) - payload[ 'name' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + validated_payload[ 'name' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) #TODO:?? if sanitized != val: log.warn( 'script kiddie' ) elif key == 'deleted': if not isinstance( val, bool ): raise ValueError( 'deleted must be a boolean: %s' %( str( type( val ) ) ) ) + validated_payload[ 'deleted' ] = val elif key == 'published': - if not isinstance( payload[ 'published' ], bool ): + if not isinstance( val, bool ): raise ValueError( 'published must be a boolean: %s' %( str( type( val ) ) ) ) + validated_payload[ 'published' ] = val elif key == 'genome_build': if not ( isinstance( val, str ) or isinstance( val, unicode ) ): raise ValueError( 'genome_build must be a string: %s' %( str( type( val ) ) ) ) - payload[ 'genome_build' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + validated_payload[ 'genome_build' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) elif key == 'annotation': if not ( isinstance( val, str ) or isinstance( val, unicode ) ): raise ValueError( 'annotation must be a string or unicode: %s' %( str( type( val ) ) ) ) - payload[ 'annotation' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) - else: + validated_payload[ 'annotation' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + elif key not in valid_but_uneditable_keys: raise AttributeError( 'unknown key: %s' %( str( key ) ) ) - return payload + return validated_payload diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -1,13 +1,13 @@ """ API operations on the contents of a history. """ -import logging -from galaxy import web +from galaxy import web, util from galaxy.web.base.controller import BaseAPIController, url_for from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin, UsesHistoryMixin from galaxy.web.base.controller import UsesLibraryMixin, UsesLibraryMixinItems +import logging log = logging.getLogger( __name__ ) class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesHistoryMixin, @@ -138,11 +138,14 @@ POST /api/histories/{encoded_history_id}/contents Creates a new history content item (file, aka HistoryDatasetAssociation). """ + #TODO: copy existing, accessible hda - dataset controller, copy_datasets + #TODO: convert existing, accessible hda - model.DatasetInstance(or hda.datatype).get_converter_types from_ld_id = payload.get( 'from_ld_id', None ) - try: history = self.get_history( trans, history_id, check_ownership=True, check_accessible=False ) except Exception, e: + #TODO: no way to tell if it failed bc of perms or other (all MessageExceptions) + trans.response.status = 500 return str( e ) if from_ld_id: @@ -164,6 +167,90 @@ else: # TODO: implement other "upload" methods here. - trans.response.status = 403 + trans.response.status = 501 return "Not implemented." + @web.expose_api + def update( self, trans, history_id, id, payload, **kwd ): + """ + PUT /api/histories/{encoded_history_id}/contents/{encoded_content_id} + Changes an existing history dataset. + """ + #TODO: PUT /api/histories/{encoded_history_id} payload = { rating: rating } (w/ no security checks) + changed = {} + try: + hda = self.get_dataset( trans, id, + check_ownership=True, check_accessible=True, check_state=True ) + # validation handled here and some parsing, processing, and conversion + payload = self._validate_and_parse_update_payload( payload ) + # additional checks here (security, etc.) + changed = self.set_hda_from_dict( trans, hda, payload ) + + except Exception, exception: + log.error( 'Update of history (%s), HDA (%s) failed: %s', + history_id, id, str( exception ), exc_info=True ) + # convert to appropo HTTP code + if( isinstance( exception, ValueError ) + or isinstance( exception, AttributeError ) ): + # bad syntax from the validater/parser + trans.response.status = 400 + else: + trans.response.status = 500 + return { 'error': str( exception ) } + + return changed + + def _validate_and_parse_update_payload( self, payload ): + """ + Validate and parse incomming data payload for an HDA. + """ + # This layer handles (most of the stricter idiot proofing): + # - unknown/unallowed keys + # - changing data keys from api key to attribute name + # - protection against bad data form/type + # - protection against malicious data content + # all other conversions and processing (such as permissions, etc.) should happen down the line + + # keys listed here don't error when attempting to set, but fail silently + # this allows PUT'ing an entire model back to the server without attribute errors on uneditable attrs + valid_but_uneditable_keys = ( + 'id', 'name', 'type', 'api_type', 'model_class', 'history_id', 'hid', + 'accessible', 'purged', 'state', 'data_type', 'file_ext', 'file_size', 'misc_blurb', + 'download_url', 'visualizations', 'display_apps', 'display_types', + 'metadata_dbkey', 'metadata_column_names', 'metadata_column_types', 'metadata_columns', + 'metadata_comment_lines', 'metadata_data_lines' + ) + + validated_payload = {} + for key, val in payload.items(): + # TODO: lots of boilerplate here, but overhead on abstraction is equally onerous + # typecheck, parse, remap key + if key == 'name': + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + raise ValueError( 'name must be a string or unicode: %s' %( str( type( val ) ) ) ) + validated_payload[ 'name' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + #TODO:?? if sanitized != val: log.warn( 'script kiddie' ) + elif key == 'deleted': + if not isinstance( val, bool ): + raise ValueError( 'deleted must be a boolean: %s' %( str( type( val ) ) ) ) + validated_payload[ 'deleted' ] = val + elif key == 'visible': + if not isinstance( val, bool ): + raise ValueError( 'visible must be a boolean: %s' %( str( type( val ) ) ) ) + validated_payload[ 'visible' ] = val + elif key == 'genome_build': + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + raise ValueError( 'genome_build must be a string: %s' %( str( type( val ) ) ) ) + validated_payload[ 'dbkey' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + elif key == 'annotation': + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + raise ValueError( 'annotation must be a string or unicode: %s' %( str( type( val ) ) ) ) + validated_payload[ 'annotation' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + elif key == 'misc_info': + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + raise ValueError( 'misc_info must be a string or unicode: %s' %( str( type( val ) ) ) ) + validated_payload[ 'info' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + elif key not in valid_but_uneditable_keys: + raise AttributeError( 'unknown key: %s' %( str( key ) ) ) + return validated_payload + diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 test/casperjs/api-hda-tests.js --- /dev/null +++ b/test/casperjs/api-hda-tests.js @@ -0,0 +1,406 @@ +/* Utility to load a specific page and output html, page text, or a screenshot + * Optionally wait for some time, text, or dom selector + */ +try { + //...if there's a better way - please let me know, universe + var scriptDir = require( 'system' ).args[3] + // remove the script filename + .replace( /[\w|\.|\-|_]*$/, '' ) + // if given rel. path, prepend the curr dir + .replace( /^(?!\/)/, './' ), + spaceghost = require( scriptDir + 'spaceghost' ).create({ + // script options here (can be overridden by CLI) + //verbose: true, + //logLevel: debug, + scriptDir: scriptDir + }); + +} catch( error ){ + console.debug( error ); + phantom.exit( 1 ); +} +spaceghost.start(); + +// =================================================================== SET UP +var utils = require( 'utils' ); + +var email = spaceghost.user.getRandomEmail(), + password = '123456'; +if( spaceghost.fixtureData.testUser ){ + email = spaceghost.fixtureData.testUser.email; + password = spaceghost.fixtureData.testUser.password; +} +spaceghost.user.loginOrRegisterUser( email, password ); + +var uploadFilename = '1.sam', + uploadFilepath = '../../test-data/' + uploadFilename, + upload = {}; +spaceghost.thenOpen( spaceghost.baseUrl ).tools.uploadFile( uploadFilepath, function( uploadInfo ){ + upload = uploadInfo; +}); + +function hasKeys( object, keysArray ){ + if( !utils.isObject( object ) ){ return false; } + for( var i=0; i<keysArray.length; i += 1 ){ + if( !object.hasOwnProperty( keysArray[i] ) ){ + spaceghost.debug( 'object missing key: ' + keysArray[i] ); + return false; + } + } + return true; +} + +function countKeys( object ){ + if( !utils.isObject( object ) ){ return 0; } + var count = 0; + for( var key in object ){ + if( object.hasOwnProperty( key ) ){ count += 1; } + } + return count; +} + +// =================================================================== TESTS +var summaryKeys = [ 'id', 'name', 'type', 'url' ], + detailKeys = [ + // the following are always present regardless of datatype + 'id', 'name', 'api_type', 'model_class', + 'history_id', 'hid', + 'accessible', 'deleted', 'visible', 'purged', + 'state', 'data_type', 'file_ext', 'file_size', + 'misc_info', 'misc_blurb', + 'download_url', 'visualizations', 'display_apps', 'display_types', + 'genome_build', + // the following are NOT always present DEPENDING ON datatype + 'metadata_dbkey', + 'metadata_column_names', 'metadata_column_types', 'metadata_columns', + 'metadata_comment_lines', 'metadata_data_lines' + ]; + +spaceghost.historypanel.waitForHdas().then( function(){ + + var uploaded = this.historypanel.hdaElementInfoByTitle( uploadFilename ); + this.info( 'found uploaded hda: ' + uploaded.attributes.id ); + this.debug( 'uploaded hda: ' + this.jsonStr( uploaded ) ); + // ------------------------------------------------------------------------------------------- INDEX + this.test.comment( 'index should return a list of summary data for each hda' ); + var histories = this.api.histories.index(), + lastHistory = histories[0], + hdaIndex = this.api.hdas.index( lastHistory.id ); + //this.debug( 'hdaIndex:' + this.jsonStr( hdaIndex ) ); + + this.test.assert( utils.isArray( hdaIndex ), "index returned an array: length " + hdaIndex.length ); + this.test.assert( hdaIndex.length >= 1, 'Has at least one hda' ); + + var firstHda = hdaIndex[0]; + this.test.assert( hasKeys( firstHda, summaryKeys ), 'Has the proper keys' ); + + this.test.assert( this.api.isEncodedId( firstHda.id ), 'Id appears well-formed: ' + firstHda.id ); + this.test.assert( uploaded.text.indexOf( firstHda.name ) !== -1, 'Title matches: ' + firstHda.name ); + // not caring about type or url here + + + // ------------------------------------------------------------------------------------------- SHOW + this.test.comment( 'show should get an HDA details object' ); + var hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + //this.debug( this.jsonStr( hdaShow ) ); + this.test.assert( hasKeys( hdaShow, detailKeys ), 'Has the proper keys' ); + + //TODO: validate data in each hdaShow attribute... + + + // ------------------------------------------------------------------------------------------- INDEX (detailed) + this.test.comment( 'index should return a list of detailed data for each hda in "ids" when passed' ); + hdaIndex = this.api.hdas.index( lastHistory.id, [ firstHda.id ] ); + this.debug( 'hdaIndex:' + this.jsonStr( hdaIndex ) ); + + this.test.assert( utils.isArray( hdaIndex ), "index returned an array: length " + hdaIndex.length ); + this.test.assert( hdaIndex.length >= 1, 'Has at least one hda' ); + + firstHda = hdaIndex[0]; + this.test.assert( hasKeys( firstHda, detailKeys ), 'Has the proper keys' ); + + //TODO??: validate data in firstHda attribute? we ASSUME it's from a common method as show... + + + // ------------------------------------------------------------------------------------------- CREATE + //TODO: create from_ld_id + + + // ------------------------------------------------------------------------------------------- UPDATE + // ........................................................................................... idiot proofing + this.test.comment( 'updating to the current value should return no value (no change)' ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + var returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + name : hdaShow.name + }); + this.test.assert( countKeys( returned ) === 0, "No changed returned: " + this.jsonStr( returned ) ); + + this.test.comment( 'updating using a nonsense key should fail with an error' ); + var err = {}; + try { + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + konamiCode : 'uuddlrlrba' + }); + } catch( error ){ + err = error; + //this.debug( this.jsonStr( err ) ); + } + this.test.assert( !!err.message, "Error occurred: " + err.message ); + this.test.assert( err.status === 400, "Error status is 400: " + err.status ); + + this.test.comment( 'updating by attempting to change type should cause an error' ); + err = {}; + try { + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + //name : false + deleted : 'sure why not' + }); + } catch( error ){ + err = error; + //this.debug( this.jsonStr( err ) ); + } + this.test.assert( !!err.message, "Error occurred: " + err.message ); + this.test.assert( err.status === 400, "Error status is 400: " + err.status ); + //TODO??: other type checks? + + + // ........................................................................................... name + this.test.comment( 'update should allow changing the name' ); + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + name : 'New name' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.name === 'New name', "Name successfully set via update: " + hdaShow.name ); + + this.test.comment( 'update should sanitize any new name' ); + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + name : 'New name<script type="text/javascript" src="bler">alert("blah");</script>' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.name === 'New name', "Update sanitized name: " + hdaShow.name ); + + //NOTE!: this fails on sqlite3 (with default setup) + try { + this.test.comment( 'update should allow unicode in names' ); + var unicodeName = 'Ржевский сапоги'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + name : unicodeName + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.name === unicodeName, "Update accepted unicode name: " + hdaShow.name ); + } catch( err ){ + //this.debug( this.jsonStr( err ) ); + if( ( err instanceof this.api.APIError ) + && ( err.status === 500 ) + && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ + this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); + } + } + + this.test.comment( 'update should allow escaped quotations in names' ); + var quotedName = '"Bler"'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + name : quotedName + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.name === quotedName, + "Update accepted escaped quotations in name: " + hdaShow.name ); + + + // ........................................................................................... deleted + this.test.comment( 'update should allow changing the deleted flag' ); + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + deleted: true + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.deleted === true, "Update set the deleted flag: " + hdaShow.deleted ); + + this.test.comment( 'update should allow changing the deleted flag back' ); + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + deleted: false + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.deleted === false, "Update set the deleted flag: " + hdaShow.deleted ); + + + // ........................................................................................... visible/hidden + this.test.comment( 'update should allow changing the visible flag' ); + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + visible: false + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.visible === false, "Update set the visible flag: " + hdaShow.visible ); + + + // ........................................................................................... genome_build/dbkey + this.test.comment( 'update should allow changing the genome_build' ); + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + genome_build : 'hg18' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.genome_build === 'hg18', + "genome_build successfully set via update: " + hdaShow.genome_build ); + this.test.assert( hdaShow.metadata_dbkey === 'hg18', + "metadata_dbkey successfully set via the same update: " + hdaShow.metadata_dbkey ); + + this.test.comment( 'update should sanitize any genome_build' ); + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + genome_build : 'hg18<script type="text/javascript" src="bler">alert("blah");</script>' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.genome_build === 'hg18', + "Update sanitized genome_build: " + hdaShow.genome_build ); + this.test.assert( hdaShow.metadata_dbkey === 'hg18', + "metadata_dbkey successfully set via the same update: " + hdaShow.metadata_dbkey ); + + this.test.comment( 'update should allow unicode in genome builds' ); + var unicodeBuild = 'Ржевский18'; + //NOTE!: this fails on sqlite3 (with default setup) + try { + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + name : unicodeBuild + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.genome_build === unicodeBuild, + "Update accepted unicode genome_build: " + hdaShow.name ); + } catch( err ){ + //this.debug( this.jsonStr( err ) ); + if( ( err instanceof this.api.APIError ) + && ( err.status === 500 ) + && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ + this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); + } + } + + // ........................................................................................... misc_info/info + this.test.comment( 'update should allow changing the misc_info' ); + var newInfo = 'I\'ve made a huge mistake.'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + misc_info : newInfo + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.misc_info === newInfo, + "misc_info successfully set via update: " + hdaShow.misc_info ); + + this.test.comment( 'update should sanitize any misc_info' ); + var newInfo = 'You\'re going to get hop-ons.'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + misc_info : newInfo + '<script type="text/javascript" src="bler">alert("blah");</script>' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.misc_info === newInfo, + "Update sanitized misc_info: " + hdaShow.misc_info ); + + this.test.comment( 'update should allow unicode in misc_info' ); + var unicodeInfo = '여보!'; + //NOTE!: this fails on sqlite3 (with default setup) + try { + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + misc_info : unicodeInfo + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.misc_info === unicodeInfo, + "Update accepted unicode misc_info: " + hdaShow.misc_info ); + } catch( err ){ + //this.debug( this.jsonStr( err ) ); + if( ( err instanceof this.api.APIError ) + && ( err.status === 500 ) + && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ + this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); + } + } + +/* + // ........................................................................................... annotation + // currently fails because no annotation is returned in details + this.test.comment( 'update should allow changing the annotation' ); + var newAnnotation = 'Found this sample on a movie theatre floor'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + annotation : newAnnotation + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.annotation === newAnnotation, + "Annotation successfully set via update: " + hdaShow.annotation ); + + this.test.comment( 'update should sanitize any new annotation' ); + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + annotation : 'New annotation<script type="text/javascript" src="bler">alert("blah");</script>' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.annotation === 'New annotation', + "Update sanitized annotation: " + hdaShow.annotation ); + + //NOTE!: this fails on sqlite3 (with default setup) + try { + this.test.comment( 'update should allow unicode in annotations' ); + var unicodeAnnotation = 'お願いは、それが落下させない'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + annotation : unicodeAnnotation + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.annotation === unicodeAnnotation, + "Update accepted unicode annotation: " + hdaShow.annotation ); + } catch( err ){ + //this.debug( this.jsonStr( err ) ); + if( ( err instanceof this.api.APIError ) + && ( err.status === 500 ) + && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ + this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); + } + } + + this.test.comment( 'update should allow escaped quotations in annotations' ); + var quotedAnnotation = '"Bler"'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + annotation : quotedAnnotation + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.annotation === quotedAnnotation, + "Update accepted escaped quotations in annotation: " + hdaShow.annotation ); +*/ + + + // ------------------------------------------------------------------------------------------- ERRORS + this.test.comment( 'create should error with "not implemented" when the param "from_ld_id" is not used' ); + var errored = false; + try { + // sending an empty object won't work + var created = this.api.hdas.create( lastHistory.id, { bler: 'bler' } ); + + } catch( err ){ + errored = true; + this.test.assert( err.message.indexOf( 'Not implemented' ) !== -1, + 'Error has the proper message: ' + err.message ); + this.test.assert( err.status === 501, 'Error has the proper status code: ' + err.status ); + } + if( !errored ){ + this.test.fail( 'create without "from_ld_id" did not cause error' ); + } + + + //var returned = this.api.hdas.update( lastHistory.id, hdaIndex[0].id, { deleted: true, blerp: 'blerp' }); + //var returned = this.api.hdas.update( lastHistory.id, { deleted: true, blerp: 'blerp' }); + //this.debug( 'returned:' + this.jsonStr( returned ) ); + //this.debug( 'page:' + this.jsonStr( this.page ) ); +}); + +// =================================================================== +spaceghost.run( function(){ +}); diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 test/casperjs/api-history-tests.js --- a/test/casperjs/api-history-tests.js +++ b/test/casperjs/api-history-tests.js @@ -36,7 +36,10 @@ function hasKeys( object, keysArray ){ if( !utils.isObject( object ) ){ return false; } for( var i=0; i<keysArray.length; i += 1 ){ - if( !object.hasOwnProperty( keysArray[i] ) ){ return false; } + if( !object.hasOwnProperty( keysArray[i] ) ){ + spaceghost.debug( 'object missing key: ' + keysArray[i] ); + return false; + } } return true; } @@ -362,6 +365,10 @@ "Update accepted escaped quotations in annotation: " + historyShow.annotation ); + // ------------------------------------------------------------------------------------------- ERRORS + //TODO: make sure expected errors are being passed back (but no permissions checks here - different suite) + // bad ids: index, show, update, delete, undelete + /* */ //this.debug( this.jsonStr( historyShow ) ); diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 test/casperjs/modules/api.js --- a/test/casperjs/modules/api.js +++ b/test/casperjs/modules/api.js @@ -232,7 +232,7 @@ }; HDAAPI.prototype.show = function show( historyId, id, deleted ){ - this.api.spaceghost.info( 'hda.show: ' + [ id, (( deleted )?( 'w deleted' ):( '' )) ] ); + this.api.spaceghost.info( 'hda.show: ' + [ historyId, id, (( deleted )?( 'w deleted' ):( '' )) ] ); id = ( id === 'most_recently_used' )?( id ):( this.api.ensureId( id ) ); deleted = deleted || false; @@ -242,7 +242,7 @@ }; HDAAPI.prototype.create = function create( historyId, payload ){ - this.api.spaceghost.info( 'hda.create: ' + this.api.spaceghost.jsonStr( payload ) ); + this.api.spaceghost.info( 'hda.create: ' + [ historyId, this.api.spaceghost.jsonStr( payload ) ] ); // py.payload <-> ajax.data payload = this.api.ensureObject( payload ); @@ -253,8 +253,7 @@ }; HDAAPI.prototype.update = function create( historyId, id, payload ){ - this.api.spaceghost.info( 'hda.update: ' + historyId + ',' + id + ',' - + this.api.spaceghost.jsonStr( payload ) ); + this.api.spaceghost.info( 'hda.update: ' + [ historyId, id, this.api.spaceghost.jsonStr( payload ) ] ); // py.payload <-> ajax.data historyId = this.api.ensureId( historyId ); diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 test/casperjs/modules/tools.js --- a/test/casperjs/modules/tools.js +++ b/test/casperjs/modules/tools.js @@ -13,7 +13,7 @@ //??: circ ref? this.options = {}; /** Default amount of ms to wait for upload to finish */ - this.options.defaultUploadWait = ( 30 * 1000 ); + this.options.defaultUploadWait = ( 45 * 1000 ); this.spaceghost = spaceghost; }; exports.Tools = Tools; @@ -108,14 +108,22 @@ // wait for main panel, history reload ////NOTE!: assumes tool execution reloads the history panel - this.waitForMultipleNavigation( [ 'tool_runner/upload_async_message', 'history' ], function(){ - // debugging - this.jumpToMain( function(){ - var messageInfo = this.elementInfoOrNull( this.data.selectors.messages.all ); - this.debug( ( messageInfo )?( messageInfo.attributes['class'] + ':\n' + messageInfo.text ) - :( 'NO post upload message' ) ); - }); - }); + this.waitForMultipleNavigation( [ 'tool_runner/upload_async_message', 'history' ], + function thenAfterUploadRefreshes(){ + // debugging + this.jumpToMain( function(){ + var messageInfo = this.elementInfoOrNull( this.data.selectors.messages.all ); + this.debug( ( messageInfo )?( messageInfo.attributes['class'] + ':\n' + messageInfo.text ) + :( 'NO post upload message' ) ); + }); + }, + function timeoutWaitingForUploadRefreshes( urlsStillWaitingOn ){ + this.capture( 'upload-error.png' ) + throw new this.GalaxyError( 'Upload Error: ' + + 'timeout waiting for upload "' + filepath + '" refreshes: ' + urlsStillWaitingOn ); + }, + this.tools.options.defaultUploadWait + ); }); }; @@ -160,13 +168,19 @@ // error if an info message wasn't found spaceghost.withMainPanel( function checkUploadMessage(){ var infoInfo = spaceghost.elementInfoOrNull( this.data.selectors.messages.infolarge ); - if( ( !infoInfo ) - || ( infoInfo.text.indexOf( this.data.text.upload.success ) === -1 ) ){ - throw new this.GalaxyError( 'Upload Error: no info message uploading "' + filepath + '"' ); + if( ( infoInfo ) + && ( infoInfo.text.indexOf( this.data.text.upload.success ) !== -1 ) ){ + // safe to store these + uploadInfo.filename = filename; + uploadInfo.filepath = filepath; + + } else { + // capture any other messages on the page + var otherInfo = spaceghost.elementInfoOrNull( this.data.selectors.messages.all ), + message = ( otherInfo && otherInfo.text )?( otherInfo.text ):( '' ); + this.capture( 'upload-error.png' ) + throw new this.GalaxyError( 'Upload Error: no success message uploading "' + filepath + '": ' + message ); } - // safe to store these - uploadInfo.filename = filename; - uploadInfo.filepath = filepath; }); // the hpanel should refresh and display the uploading file, wait for that to go into the ok state @@ -177,6 +191,7 @@ if( hdaElement === null ){ var hdaContainer = this.historypanel.data.selectors.hdaContainer; this.warning( 'Upload Error: ' + hdaContainer + ':\n' + this.getHTML( hdaContainer ) ); + this.capture( 'upload-error.png' ) throw new this.GalaxyError( 'Upload Error: uploaded file HDA not found: ' + uploadInfo.filename ); } this.debug( 'uploaded HDA element: ' + this.jsonStr( this.quickInfo( hdaElement ) ) ); @@ -191,6 +206,7 @@ }, function timeoutFn( newHdaInfo ){ this.warning( 'timeout waiting for upload:\n' + this.jsonStr( this.quickInfo( newHdaInfo ) ) ); + this.capture( 'upload-error.png' ) throw new spaceghost.GalaxyError( 'Upload Error: timeout waiting for ok state: ' + '"' + uploadInfo.filepath + '" (waited ' + timeoutAfterMs + ' ms)' ); diff -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 -r 8b9ca63f9128fbe9c7f01805db64da3ec2916332 test/casperjs/spaceghost.js --- a/test/casperjs/spaceghost.js +++ b/test/casperjs/spaceghost.js @@ -546,9 +546,10 @@ * @param {String} urlToWaitFor the url to wait for (rel. to spaceghost.baseUrl) * @param {Function} then the function to call after the nav request * @param {Function} timeoutFn the function to call on timeout (optional) + * @param {Integer} waitMs manual setting of ms to wait (optional) */ -SpaceGhost.prototype.waitForNavigation = function waitForNavigation( urlToWaitFor, then, timeoutFn ){ - return this.waitForMultipleNavigation( [ urlToWaitFor ], then, timeoutFn ); +SpaceGhost.prototype.waitForNavigation = function waitForNavigation( urlToWaitFor, then, timeoutFn, waitMs ){ + return this.waitForMultipleNavigation( [ urlToWaitFor ], then, timeoutFn, waitMs ); }; /** Wait for a multiple navigation requests then call a function. @@ -557,9 +558,13 @@ * @param {String[]} urlsToWaitFor the relative urls to wait for * @param {Function} then the function to call after the nav request * @param {Function} timeoutFn the function to call on timeout (optional) + * @param {Integer} waitMs manual setting of ms to wait (optional) */ -SpaceGhost.prototype.waitForMultipleNavigation = function waitForMultipleNavigation( urlsToWaitFor, then, timeoutFn ){ - this.info( 'waiting for navigation: ' + this.jsonStr( urlsToWaitFor ) ); +SpaceGhost.prototype.waitForMultipleNavigation = function waitForMultipleNavigation( urlsToWaitFor, + then, timeoutFn, waitMs ){ + waitMs = waitMs || ( this.options.waitTimeout * urlsToWaitFor.length ); + + this.info( 'waiting for navigation: ' + this.jsonStr( urlsToWaitFor ) + ', timeout after: ' + waitMs ); function urlMatches( urlToMatch, url ){ return ( url.indexOf( spaceghost.baseUrl + '/' + urlToMatch ) !== -1 ); } @@ -589,9 +594,10 @@ if( utils.isFunction( then ) ){ then.call( this ); } }, function timeout(){ - if( utils.isFunction( timeoutFn ) ){ timeoutFn.call( this ); } + this.removeListener( 'navigation.requested', catchNavReq ); + if( utils.isFunction( timeoutFn ) ){ timeoutFn.call( this, urlsToWaitFor ); } }, - this.options.waitTimeout * urlsToWaitFor.length + waitMs ); return this; }; Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Account for the scenario where a test status is not present in the tool_test_errors dict.
by commits-noreply＠bitbucket.org 24 Apr '13

24 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2df57338a595/ Changeset: 2df57338a595 User: inithello Date: 2013-04-24 20:55:53 Summary: Account for the scenario where a test status is not present in the tool_test_errors dict. Affected #: 1 file diff -r 1a915a754396da8ac986dd410ee99bed12668235 -r 2df57338a595ab96f3f54af2b0e4c2e382198a40 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1171,9 +1171,9 @@ # Generate a citable URL for this repository with owner and changeset revision. repository_citable_url = suc.url_join( tool_shed_url, 'view', user.username, repository.name, metadata_row.changeset_revision ) title = 'Functional test results for changeset revision %s of %s' % ( metadata_row.changeset_revision, repository.name ) - tests_passed = len( metadata_row.tool_test_errors[ 'tests_passed' ] ) - tests_failed = len( metadata_row.tool_test_errors[ 'invalid_tests' ] ) - invalid_tests = len( metadata_row.tool_test_errors[ 'test_errors' ] ) + tests_passed = len( metadata_row.tool_test_errors.get( 'tests_passed', [] ) ) + tests_failed = len( metadata_row.tool_test_errors.get( 'test_errors', [] ) ) + invalid_tests = len( metadata_row.tool_test_errors.get( 'invalid_tests', [] ) ) description = '%d tests passed, %d tests failed, %d tests determined to be invalid.' % ( tests_passed, tests_failed, invalid_tests ) # The guid attribute in an RSS feed's list of items allows a feed reader to choose not to show an item as updated # if the guid is unchanged. For functional test results, the citable URL is sufficiently unique to enable Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add support for a new <action type="set_environmnet_for_install"> tag in tool_dependencies.xml files included in tool shed repositories. This tag currently can include any number of <repository> tags, each of which will contain any number of tool dependency tags (i.e., <package> or <set_environmnet> tags). The settings in the env.sh file for each of the tool dependency tags will be injected into the environment for all following shell commands defined in the too
by commits-noreply＠bitbucket.org 24 Apr '13

24 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1a915a754396/ Changeset: 1a915a754396 User: greg Date: 2013-04-24 20:46:54 Summary: Add support for a new <action type="set_environmnet_for_install"> tag in tool_dependencies.xml files included in tool shed repositories. This tag currently can include any number of <repository> tags, each of which will contain any number of tool dependency tags (i.e., <package> or <set_environmnet> tags). The settings in the env.sh file for each of the tool dependency tags will be injected into the environment for all following shell commands defined in the tool_dependencies.xml file, ensuring that the required tool dependencies are sued when compiling the current dependency. I'm sure this will make no sense to anyone reading this commit message. Affected #: 7 files diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3443,7 +3443,7 @@ """Return the repository's tool dependencies that are currently installed.""" installed_dependencies = [] for tool_dependency in self.tool_dependencies: - if tool_dependency.status == ToolDependency.installation_status.INSTALLED: + if tool_dependency.status in [ ToolDependency.installation_status.INSTALLED, ToolDependency.installation_status.ERROR ]: installed_dependencies.append( tool_dependency ) return installed_dependencies @property diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -478,7 +478,7 @@ tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir )#relative_install_dir ) + tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir ) installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -1,6 +1,14 @@ -import os, shutil, tarfile, urllib2, zipfile +import logging +import os +import shutil +import tarfile +import urllib2 +import zipfile +import tool_shed.util.shed_util_common as suc from galaxy.datatypes import checkers +log = logging.getLogger( __name__ ) + def create_env_var_dict( elem, tool_dependency_install_dir=None, tool_shed_repository_install_dir=None ): env_var_name = elem.get( 'name', 'PATH' ) env_var_action = elem.get( 'action', 'prepend_to' ) @@ -76,6 +84,67 @@ zip_archive.close() return True +def get_env_shell_file_path( installation_directory ): + env_shell_file_name = 'env.sh' + default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) + if os.path.exists( default_location ): + return default_location + for root, dirs, files in os.walk( installation_directory ): + for name in files: + if name == env_shell_file_name: + return os.path.abspath( os.path.join( root, name ) ) + return None + +def get_env_shell_file_paths( app, elem ): + # Currently only the following tag set is supported. + # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> + # <package name="numpy" version="1.7.1" /> + # </repository> + env_shell_file_paths = [] + toolshed = elem.get( 'toolshed', None ) + repository_name = elem.get( 'name', None ) + repository_owner = elem.get( 'owner', None ) + changeset_revision = elem.get( 'changeset_revision', None ) + if toolshed and repository_name and repository_owner and changeset_revision: + repository = suc.get_repository_for_dependency_relationship( app, toolshed, repository_name, repository_owner, changeset_revision ) + if repository: + for sub_elem in elem: + tool_dependency_type = sub_elem.tag + tool_dependency_name = sub_elem.get( 'name' ) + tool_dependency_version = sub_elem.get( 'version' ) + if tool_dependency_type and tool_dependency_name and tool_dependency_version: + # Get the tool_dependency so we can get it's installation directory. + tool_dependency = None + for tool_dependency in repository.tool_dependencies: + if tool_dependency.type == tool_dependency_type and tool_dependency.name == tool_dependency_name and tool_dependency.version == tool_dependency_version: + break + if tool_dependency: + tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) + installation_directory = tool_dependency.installation_directory( app ) + env_shell_file_path = get_env_shell_file_path( installation_directory ) + env_shell_file_paths.append( env_shell_file_path ) + else: + error_message = "Skipping tool dependency definition because unable to locate tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " + error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + else: + error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + return env_shell_file_paths + def isbz2( file_path ): return checkers.is_bz2( file_path ) diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -28,30 +28,18 @@ if int( version.split( "." )[ 0 ] ) < 1: raise NotImplementedError( "Install Fabric version 1.0 or later." ) -def set_galaxy_environment( galaxy_user, tool_dependency_dir, host='localhost', shell='/bin/bash -l -c' ): - """General Galaxy environment configuration""" - env.user = galaxy_user - env.install_dir = tool_dependency_dir - env.host_string = host - env.shell = shell - env.use_sudo = False - env.safe_cmd = local - return env - -@contextmanager -def make_tmp_dir(): - work_dir = tempfile.mkdtemp() - yield work_dir - if os.path.exists( work_dir ): - local( 'rm -rf %s' % work_dir ) - def handle_command( app, tool_dependency, install_dir, cmd ): sa_session = app.model.context.current output = local( cmd, capture=True ) log_results( cmd, output, os.path.join( install_dir, INSTALLATION_LOG ) ) if output.return_code: tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = str( output.stderr ) + if output.stderr: + tool_dependency.error_message = str( output.stderr )[ :32768 ] + elif output.stdout: + tool_dependency.error_message = str( output.stdout )[ :32768 ] + else: + tool_dependency.error_message = "Unknown error occurred executing shell command %s, return_code: %s" % ( str( cmd ), str( output.return_code ) ) sa_session.add( tool_dependency ) sa_session.flush() return output.return_code @@ -63,6 +51,7 @@ package_name = actions_dict[ 'package_name' ] actions = actions_dict.get( 'actions', None ) filtered_actions = [] + env_shell_file_paths = [] if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): @@ -133,9 +122,19 @@ return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return + elif action_type == 'set_environment_for_install': + # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, + # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> + # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. + env_shell_file_paths = action_dict[ 'env_shell_file_paths' ] elif action_type == 'shell_command': with settings( warn_only=True ): - return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) + cmd = '' + for env_shell_file_path in env_shell_file_paths: + for i, env_setting in enumerate( open( env_shell_file_path ) ): + cmd += '%s\n' % env_setting + cmd += action_dict[ 'command' ] + return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return @@ -157,3 +156,20 @@ logfile.write( str( fabric_AttributeString.stderr ) ) logfile.write( "\n#############################################\n" ) logfile.close() + +@contextmanager +def make_tmp_dir(): + work_dir = tempfile.mkdtemp() + yield work_dir + if os.path.exists( work_dir ): + local( 'rm -rf %s' % work_dir ) + +def set_galaxy_environment( galaxy_user, tool_dependency_dir, host='localhost', shell='/bin/bash -l -c' ): + """General Galaxy environment configuration. This method is not currently used.""" + env.user = galaxy_user + env.install_dir = tool_dependency_dir + env.host_string = host + env.shell = shell + env.use_sudo = False + env.safe_cmd = local + return env diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -282,9 +282,6 @@ tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add( tool_dependency ) sa_session.flush() - - - else: package_install_version = package_elem.get( 'version', '1.0' ) tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, @@ -378,7 +375,28 @@ action_dict[ env_elem.tag ] = env_var_dicts else: continue + elif action_type == 'set_environment_for_install': + # <action type="set_environment_for_install"> + # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> + # <package name="numpy" version="1.7.1" /> + # </repository> + # </action> + # This action type allows for defining an environment that will properly compile a tool dependency. Currently, tag set definitions like + # that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be + # supported. The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current + # tool dependency package. See the package_matplotlib_1_2 repository in the test tool shed for a real-world example. + all_env_shell_file_paths = [] + for env_elem in action_elem: + if env_elem.tag == 'repository': + env_shell_file_paths = common_util.get_env_shell_file_paths( app, env_elem ) + if env_shell_file_paths: + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + else: + continue else: + log.debug( "Skipping unsupported action type '%s'." % str( action_type ) ) continue actions.append( ( action_type, action_dict ) ) if actions: diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -307,6 +307,7 @@ will be installed in: ~/<app.config.tool_dependency_dir>/<package_name>/<package_version>/<repo_owner>/<repo_name>/<repo_installed_changeset_revision> """ + sa_session = app.model.context.current installed_tool_dependencies = [] # Parse the tool_dependencies.xml config. try: @@ -327,12 +328,30 @@ if tool_dependency.name==package_name and tool_dependency.version==package_version: break if tool_dependency.can_install: - tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) + try: + tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) + except Exception, e: + error_message = "Error installing tool dependency %s version %s: %s" % ( str( package_name ), str( package_version ), str( e ) ) + log.debug( error_message ) + if tool_dependency: + tool_dependency.status = app.model.ToolDependency.installation_status.ERROR + tool_dependency.error_message = error_message + sa_session.add( tool_dependency ) + sa_session.flush() if tool_dependency and tool_dependency.status in [ app.model.ToolDependency.installation_status.INSTALLED, app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) elif elem.tag == 'set_environment': - tool_dependency = set_environment( app, elem, tool_shed_repository ) + try: + tool_dependency = set_environment( app, elem, tool_shed_repository ) + except Exception, e: + error_message = "Error setting environment for tool dependency: %s" % str( e ) + log.debug( error_message ) + if tool_dependency: + tool_dependency.status = app.model.ToolDependency.installation_status.ERROR + tool_dependency.error_message = error_message + sa_session.add( tool_dependency ) + sa_session.flush() if tool_dependency and tool_dependency.status in [ app.model.ToolDependency.installation_status.INSTALLED, app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -32,7 +32,8 @@ log = logging.getLogger( __name__ ) INITIAL_CHANGELOG_HASH = '000000000000' -MAX_CONTENT_SIZE = 32768 +MAX_CONTENT_SIZE = 1048576 +MAX_DISPLAY_SIZE = 32768 VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}&<>" ) new_repo_email_alert_template = """ @@ -667,10 +668,17 @@ safe_str = '' for i, line in enumerate( open( file_path ) ): safe_str = '%s%s' % ( safe_str, to_safe_string( line ) ) + # Stop reading after string is larger than MAX_CONTENT_SIZE. if len( safe_str ) > MAX_CONTENT_SIZE: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) - safe_str = '%s%s' % ( safe_str, to_safe_string( large_str ) ) + large_str = \ + to_safe_string( '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) ) + safe_str = '%s%s' % ( safe_str, large_str ) break + if len( safe_str ) > MAX_DISPLAY_SIZE: + # Eliminate the middle of the file to display a file no larger than MAX_DISPLAY_SIZE. This may not be ideal if the file is larger than MAX_CONTENT_SIZE. + join_by_str = \ + to_safe_string( "\n\n...some text eliminated here because file size is larger than maximum viewing size of %s...\n\n" % util.nice_size( MAX_DISPLAY_SIZE ) ) + safe_str = util.shrink_string_by_size( safe_str, MAX_DISPLAY_SIZE, join_by=join_by_str, left_larger=True, beginning_on_size_error=True ) return safe_str def get_repository_files( trans, folder_path ): @@ -787,9 +795,8 @@ This method assumes all repository tools are defined in a single shed-related tool panel config. """ tool_shed = clean_tool_shed_url( repository.tool_shed ) - partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) + relative_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) # Get the relative tool installation paths from each of the shed tool configs. - relative_install_dir = None shed_config_dict = repository.get_shed_config_dict( app ) if not shed_config_dict: # Just pick a semi-random shed config. @@ -799,7 +806,6 @@ break shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] - relative_install_dir = partial_install_dir return shed_tool_conf, tool_path, relative_install_dir def get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf ): @@ -1188,13 +1194,13 @@ return toolshed_base_url.rstrip( '/' ) == str( url_for( '/', qualified=True ) ).rstrip( '/' ) def translate_string( raw_text, to_html=True ): - """Return a subset of a string (up to MAX_CONTENT_SIZE) translated to a safe string for display in a browser.""" + """Return a subset of a string (up to MAX_DISPLAY_SIZE) translated to a safe string for display in a browser.""" if raw_text: - if len( raw_text ) <= MAX_CONTENT_SIZE: + if len( raw_text ) <= MAX_DISPLAY_SIZE: translated_string = to_safe_string( raw_text, to_html=to_html ) else: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) - translated_string = to_safe_string( '%s%s' % ( raw_text[ 0:MAX_CONTENT_SIZE ], large_str ), to_html=to_html ) + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_DISPLAY_SIZE ) + translated_string = to_safe_string( '%s%s' % ( raw_text[ 0:MAX_DISPLAY_SIZE ], large_str ), to_html=to_html ) else: translated_string = '' return translated_string Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Retrieve functional test results by owner's username instead of user ID.
by commits-noreply＠bitbucket.org 24 Apr '13

24 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/333dfc42a868/ Changeset: 333dfc42a868 User: inithello Date: 2013-04-24 20:41:16 Summary: Retrieve functional test results by owner's username instead of user ID. Affected #: 1 file diff -r 322d7ae99729ac4235e3f6becc6707c9f0c359d1 -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1127,14 +1127,17 @@ @web.expose def get_functional_test_rss( self, trans, **kwd ): - '''Return an RSS feed of the functional test results for the provided user ID, optionally filtered by the 'status' parameter.''' - encoded_user_id = kwd.get( 'user_id', None ) - if encoded_user_id: - user_id = trans.security.decode_id( encoded_user_id ) + '''Return an RSS feed of the functional test results for the provided user, optionally filtered by the 'status' parameter.''' + owner = kwd.get( 'owner', None ) + status = kwd.get( 'status', 'all' ) + if owner: + user = suc.get_user_by_username( trans.app, owner ) else: trans.response.status = 404 - return 'Unknown or missing user ID.' - status = kwd.get( 'status', 'all' ) + return 'Missing owner parameter.' + if user is None: + trans.response.status = 404 + return 'No user found with username %s.' % owner if status == 'passed': # Return only metadata revisions where tools_functionally_correct is set to True. metadata_filter = and_( trans.model.RepositoryMetadata.table.c.includes_tools == True, @@ -1150,6 +1153,7 @@ metadata_filter = and_( trans.model.RepositoryMetadata.table.c.includes_tools == True, trans.model.RepositoryMetadata.table.c.time_last_tested is not None ) + tool_shed_url = web.url_for( '/', qualified=True ) functional_test_results = [] for metadata_row in trans.sa_session.query( trans.model.RepositoryMetadata ) \ .filter( metadata_filter ) \ @@ -1157,17 +1161,15 @@ .filter( and_( trans.model.Repository.table.c.deleted == False, trans.model.Repository.table.c.private == False, trans.model.Repository.table.c.deprecated == False, - trans.model.Repository.table.c.user_id == user_id ) ): + trans.model.Repository.table.c.user_id == user.id ) ): if not metadata_row.tool_test_errors: continue # Per the RSS 2.0 specification, all dates in RSS feeds must be formatted as specified in RFC 822 # section 5.1, e.g. Sat, 07 Sep 2002 00:00:01 UT time_tested = metadata_row.time_last_tested.strftime( '%a, %d %b %Y %H:%M:%S UT' ) - link = web.url_for( '/', qualified=True ) repository = metadata_row.repository - user = repository.user # Generate a citable URL for this repository with owner and changeset revision. - repository_citable_url = suc.url_join( link, 'view', user.username, repository.name, metadata_row.changeset_revision ) + repository_citable_url = suc.url_join( tool_shed_url, 'view', user.username, repository.name, metadata_row.changeset_revision ) title = 'Functional test results for changeset revision %s of %s' % ( metadata_row.changeset_revision, repository.name ) tests_passed = len( metadata_row.tool_test_errors[ 'tests_passed' ] ) tests_failed = len( metadata_row.tool_test_errors[ 'invalid_tests' ] ) @@ -1184,7 +1186,7 @@ trans.response.set_content_type( 'application/rss+xml' ) return trans.fill_template( '/rss.mako', title='Tool functional test results', - link=link, + link=tool_shed_url, description='Functional test results for repositories owned by %s.' % user.username, pubdate=strftime( '%a, %d %b %Y %H:%M:%S UT', gmtime() ), items=functional_test_results ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Basic rss feed for tool functional test results for repositories owned by a specific user.
by commits-noreply＠bitbucket.org 24 Apr '13

24 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/322d7ae99729/ Changeset: 322d7ae99729 User: inithello Date: 2013-04-24 20:14:15 Summary: Basic rss feed for tool functional test results for repositories owned by a specific user. Affected #: 2 files diff -r 4126ec15fd614dff9d5a0b555623626fa947197c -r 322d7ae99729ac4235e3f6becc6707c9f0c359d1 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1125,6 +1125,70 @@ named_tmp_file = suc.get_named_tmpfile_from_ctx( ctx, file_name, dir ) return named_tmp_file + @web.expose + def get_functional_test_rss( self, trans, **kwd ): + '''Return an RSS feed of the functional test results for the provided user ID, optionally filtered by the 'status' parameter.''' + encoded_user_id = kwd.get( 'user_id', None ) + if encoded_user_id: + user_id = trans.security.decode_id( encoded_user_id ) + else: + trans.response.status = 404 + return 'Unknown or missing user ID.' + status = kwd.get( 'status', 'all' ) + if status == 'passed': + # Return only metadata revisions where tools_functionally_correct is set to True. + metadata_filter = and_( trans.model.RepositoryMetadata.table.c.includes_tools == True, + trans.model.RepositoryMetadata.table.c.tools_functionally_correct == True, + trans.model.RepositoryMetadata.table.c.time_last_tested is not None ) + elif status == 'failed': + # Return only metadata revisions where tools_functionally_correct is set to False. + metadata_filter = and_( trans.model.RepositoryMetadata.table.c.includes_tools == True, + trans.model.RepositoryMetadata.table.c.tools_functionally_correct == False, + trans.model.RepositoryMetadata.table.c.time_last_tested is not None ) + else: + # Return all metadata entries for this user's repositories. + metadata_filter = and_( trans.model.RepositoryMetadata.table.c.includes_tools == True, + trans.model.RepositoryMetadata.table.c.time_last_tested is not None ) + + functional_test_results = [] + for metadata_row in trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( metadata_filter ) \ + .join( trans.model.Repository ) \ + .filter( and_( trans.model.Repository.table.c.deleted == False, + trans.model.Repository.table.c.private == False, + trans.model.Repository.table.c.deprecated == False, + trans.model.Repository.table.c.user_id == user_id ) ): + if not metadata_row.tool_test_errors: + continue + # Per the RSS 2.0 specification, all dates in RSS feeds must be formatted as specified in RFC 822 + # section 5.1, e.g. Sat, 07 Sep 2002 00:00:01 UT + time_tested = metadata_row.time_last_tested.strftime( '%a, %d %b %Y %H:%M:%S UT' ) + link = web.url_for( '/', qualified=True ) + repository = metadata_row.repository + user = repository.user + # Generate a citable URL for this repository with owner and changeset revision. + repository_citable_url = suc.url_join( link, 'view', user.username, repository.name, metadata_row.changeset_revision ) + title = 'Functional test results for changeset revision %s of %s' % ( metadata_row.changeset_revision, repository.name ) + tests_passed = len( metadata_row.tool_test_errors[ 'tests_passed' ] ) + tests_failed = len( metadata_row.tool_test_errors[ 'invalid_tests' ] ) + invalid_tests = len( metadata_row.tool_test_errors[ 'test_errors' ] ) + description = '%d tests passed, %d tests failed, %d tests determined to be invalid.' % ( tests_passed, tests_failed, invalid_tests ) + # The guid attribute in an RSS feed's list of items allows a feed reader to choose not to show an item as updated + # if the guid is unchanged. For functional test results, the citable URL is sufficiently unique to enable + # that behavior. + functional_test_results.append( dict( title=title, + guid=repository_citable_url, + link=repository_citable_url, + description=description, + pubdate=time_tested ) ) + trans.response.set_content_type( 'application/rss+xml' ) + return trans.fill_template( '/rss.mako', + title='Tool functional test results', + link=link, + description='Functional test results for repositories owned by %s.' % user.username, + pubdate=strftime( '%a, %d %b %Y %H:%M:%S UT', gmtime() ), + items=functional_test_results ) + def get_metadata( self, trans, repository_id, changeset_revision ): repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) if repository_metadata and repository_metadata.metadata: diff -r 4126ec15fd614dff9d5a0b555623626fa947197c -r 322d7ae99729ac4235e3f6becc6707c9f0c359d1 templates/rss.mako --- /dev/null +++ b/templates/rss.mako @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="UTF-8"?> +<rss version="2.0"> + <channel> + <title>${title}</title> + <link>${link}</link> + <pubDate>${pubdate}</pubDate> + <description>${description}</description> + <language>en-US</language> + <ttl>60</ttl> + <docs>http://cyber.law.harvard.edu/rss/rss.html</docs> + %for item in items: + <item> + <pubDate>${item['pubdate']}</pubDate> + <title>${item['title']}</title> + <link>${item['link']}</link> + <guid>${item['guid']}</guid> + <description> + ${item['description']} + </description> + </item> + %endfor + </channel> +</rss> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 24 Apr '13

24 Apr '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4126ec15fd61/ Changeset: 4126ec15fd61 User: inithello Date: 2013-04-24 15:52:40 Summary: Fix functional test issue with forms having refresh_on_change. Affected #: 1 file diff -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 -r 4126ec15fd614dff9d5a0b555623626fa947197c test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1180,9 +1180,16 @@ # Check for refresh_on_change attribute, submit a change if required if hasattr( control, 'attrs' ) and 'refresh_on_change' in control.attrs.keys(): changed = False - item_labels = [ item.attrs[ 'label' ] for item in control.get_items() if item.selected ] #For DataToolParameter, control.value is the HDA id, but kwd contains the filename. This loop gets the filename/label for the selected values. + # For DataToolParameter, control.value is the HDA id, but kwd contains the filename. + # This loop gets the filename/label for the selected values. + item_labels = [ item.attrs[ 'label' ] for item in control.get_items() if item.selected ] for value in kwd[ control.name ]: - if value not in control.value and True not in [ value in item_label for item_label in item_labels ]: + # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py + if len( value ) > 30 and control.is_of_kind( 'singlelist' ): + field_value = '%s..%s' % ( elem[:17], elem[-11:] ) + else: + field_value = value + if field_value not in control.value and True not in [ field_value in item_label for item_label in item_labels ]: changed = True break if changed: @@ -1190,7 +1197,11 @@ control.clear() # kwd[control.name] should be a singlelist for elem in kwd[ control.name ]: - tc.fv( f.name, control.name, str( elem ) ) + if len( elem ) > 30 and control.is_of_kind( 'singlelist' ): + elem_name = '%s..%s' % ( elem[:17], elem[-11:] ) + else: + elem_name = elem + tc.fv( f.name, control.name, str( elem_name ) ) # Create a new submit control, allows form to refresh, instead of going to next page control = ClientForm.SubmitControl( 'SubmitControl', '___refresh_grouping___', {'name':'refresh_grouping'} ) control.add_to_form( f ) @@ -1241,7 +1252,7 @@ tc.fv( f.name, control.name, str( elem ) ) except Exception, e2: print "Attempting to set control '", control.name, "' to value '", elem, "' threw exception: ", e2 - # Galaxy truncates long file names in the dataset_collector in ~/parameters/basic.py + # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py if len( elem ) > 30: elem_name = '%s..%s' % ( elem[:17], elem[-11:] ) else: https://bitbucket.org/galaxy/galaxy-central/commits/b0ea9722b9dd/ Changeset: b0ea9722b9dd Branch: stable User: inithello Date: 2013-04-24 15:52:40 Summary: Fix functional test issue with forms having refresh_on_change. Affected #: 1 file diff -r ae53deed05b68f6febfcf3472c2a11514c12eb0c -r b0ea9722b9dd8e0da63208b59be20b90eae0cea7 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1180,9 +1180,16 @@ # Check for refresh_on_change attribute, submit a change if required if hasattr( control, 'attrs' ) and 'refresh_on_change' in control.attrs.keys(): changed = False - item_labels = [ item.attrs[ 'label' ] for item in control.get_items() if item.selected ] #For DataToolParameter, control.value is the HDA id, but kwd contains the filename. This loop gets the filename/label for the selected values. + # For DataToolParameter, control.value is the HDA id, but kwd contains the filename. + # This loop gets the filename/label for the selected values. + item_labels = [ item.attrs[ 'label' ] for item in control.get_items() if item.selected ] for value in kwd[ control.name ]: - if value not in control.value and True not in [ value in item_label for item_label in item_labels ]: + # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py + if len( value ) > 30 and control.is_of_kind( 'singlelist' ): + field_value = '%s..%s' % ( elem[:17], elem[-11:] ) + else: + field_value = value + if field_value not in control.value and True not in [ field_value in item_label for item_label in item_labels ]: changed = True break if changed: @@ -1190,7 +1197,11 @@ control.clear() # kwd[control.name] should be a singlelist for elem in kwd[ control.name ]: - tc.fv( f.name, control.name, str( elem ) ) + if len( elem ) > 30 and control.is_of_kind( 'singlelist' ): + elem_name = '%s..%s' % ( elem[:17], elem[-11:] ) + else: + elem_name = elem + tc.fv( f.name, control.name, str( elem_name ) ) # Create a new submit control, allows form to refresh, instead of going to next page control = ClientForm.SubmitControl( 'SubmitControl', '___refresh_grouping___', {'name':'refresh_grouping'} ) control.add_to_form( f ) @@ -1241,7 +1252,7 @@ tc.fv( f.name, control.name, str( elem ) ) except Exception, e2: print "Attempting to set control '", control.name, "' to value '", elem, "' threw exception: ", e2 - # Galaxy truncates long file names in the dataset_collector in ~/parameters/basic.py + # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py if len( elem ) > 30: elem_name = '%s..%s' % ( elem[:17], elem[-11:] ) else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: History API: add update method and allow name, genome_build, annotation, deleted, and published as updatable fields; browser tests: test history api; root/history: fix error handling when user is anonymous
by commits-noreply＠bitbucket.org 23 Apr '13

23 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fb28ceb83c37/ Changeset: fb28ceb83c37 User: carlfeberhard Date: 2013-04-23 23:38:00 Summary: History API: add update method and allow name, genome_build, annotation, deleted, and published as updatable fields; browser tests: test history api; root/history: fix error handling when user is anonymous Affected #: 9 files diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -575,8 +575,10 @@ self.group = group class History( object, UsesAnnotations ): + api_collection_visible_keys = ( 'id', 'name', 'published', 'deleted' ) - api_element_visible_keys = ( 'id', 'name', 'published', 'deleted' ) + api_element_visible_keys = ( 'id', 'name', 'published', 'deleted', 'genome_build', 'purged' ) + def __init__( self, id=None, name=None, user=None ): self.id = id self.name = name or "Unnamed history" @@ -589,6 +591,7 @@ self.user = user self.datasets = [] self.galaxy_sessions = [] + def _next_hid( self ): # TODO: override this with something in the database that ensures # better integrity @@ -600,18 +603,21 @@ if dataset.hid > last_hid: last_hid = dataset.hid return last_hid + 1 + def add_galaxy_session( self, galaxy_session, association=None ): if association is None: self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) ) else: self.galaxy_sessions.append( association ) + def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid=True, quota=True ): if isinstance( dataset, Dataset ): dataset = HistoryDatasetAssociation(dataset=dataset) object_session( self ).add( dataset ) object_session( self ).flush() elif not isinstance( dataset, HistoryDatasetAssociation ): - raise TypeError, "You can only add Dataset and HistoryDatasetAssociation instances to a history ( you tried to add %s )." % str( dataset ) + raise TypeError, ( "You can only add Dataset and HistoryDatasetAssociation instances to a history" + + " ( you tried to add %s )." % str( dataset ) ) if parent_id: for data in self.datasets: if data.id == parent_id: @@ -630,6 +636,7 @@ self.genome_build = genome_build self.datasets.append( dataset ) return dataset + def copy( self, name=None, target_user=None, activatable=False ): # Create new history. if not name: @@ -647,7 +654,7 @@ # Copy annotation. self.copy_item_annotation( db_session, self.user, self, target_user, new_history ) - #Copy Tags + # Copy Tags new_history.copy_tags_from(target_user=target_user, source_history=self) # Copy HDAs. @@ -667,12 +674,17 @@ db_session.add( new_history ) db_session.flush() return new_history + @property def activatable_datasets( self ): # This needs to be a list return [ hda for hda in self.datasets if not hda.dataset.deleted ] + def get_display_name( self ): - """ History name can be either a string or a unicode object. If string, convert to unicode object assuming 'utf-8' format. """ + """ + History name can be either a string or a unicode object. + If string, convert to unicode object assuming 'utf-8' format. + """ history_name = self.name if isinstance(history_name, str): history_name = unicode(history_name, 'utf-8') @@ -682,6 +694,7 @@ if value_mapper is None: value_mapper = {} rval = {} + try: visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) except AttributeError: @@ -693,6 +706,7 @@ rval[key] = value_mapper.get( key )( rval[key] ) except AttributeError: rval[key] = None + tags_str_list = [] for tag in self.tags: tag_str = tag.user_tname @@ -702,25 +716,51 @@ rval['tags'] = tags_str_list rval['model_class'] = self.__class__.__name__ return rval + + def set_from_dict( self, new_data ): + #AKA: set_api_value + """ + Set object attributes to the values in dictionary new_data limiting + to only those keys in api_element_visible_keys. + + Returns a dictionary of the keys, values that have been changed. + """ + # precondition: keys are proper, values are parsed and validated + changed = {} + for key in [ k for k in new_data.keys() if k in self.api_element_visible_keys ]: + new_val = new_data[ key ] + old_val = self.__getattribute__( key ) + if new_val == old_val: + continue + + self.__setattr__( key, new_val ) + changed[ key ] = new_val + + return changed + @property def get_disk_size_bytes( self ): return self.get_disk_size( nice_size=False ) + def unhide_datasets( self ): for dataset in self.datasets: dataset.mark_unhidden() + def resume_paused_jobs( self ): for dataset in self.datasets: job = dataset.creating_job if job is not None and job.state == Job.states.PAUSED: job.set_state(Job.states.NEW) + def get_disk_size( self, nice_size=False ): # unique datasets only db_session = object_session( self ) - rval = db_session.query( func.sum( db_session.query( HistoryDatasetAssociation.dataset_id, Dataset.total_size ).join( Dataset ) - .filter( HistoryDatasetAssociation.table.c.history_id == self.id ) - .filter( HistoryDatasetAssociation.purged != True ) - .filter( Dataset.purged != True ) - .distinct().subquery().c.total_size ) ).first()[0] + rval = db_session.query( + func.sum( db_session.query( HistoryDatasetAssociation.dataset_id, Dataset.total_size ).join( Dataset ) + .filter( HistoryDatasetAssociation.table.c.history_id == self.id ) + .filter( HistoryDatasetAssociation.purged != True ) + .filter( Dataset.purged != True ) + .distinct().subquery().c.total_size ) ).first()[0] if rval is None: rval = 0 if nice_size: @@ -733,6 +773,7 @@ new_shta.user = target_user self.tags.append(new_shta) + class HistoryUserShareAssociation( object ): def __init__( self ): self.history = None diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -245,6 +245,187 @@ return item +class UsesHistoryMixin( SharableItemSecurityMixin ): + """ Mixin for controllers that use History objects. """ + + def get_history( self, trans, id, check_ownership=True, check_accessible=False, deleted=None ): + """Get a History from the database by id, verifying ownership.""" + history = self.get_object( trans, id, 'History', check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted ) + return self.security_check( trans, history, check_ownership, check_accessible ) + + def get_history_datasets( self, trans, history, show_deleted=False, show_hidden=False, show_purged=False ): + """ Returns history's datasets. """ + query = trans.sa_session.query( trans.model.HistoryDatasetAssociation ) \ + .filter( trans.model.HistoryDatasetAssociation.history == history ) \ + .options( eagerload( "children" ) ) \ + .join( "dataset" ) \ + .options( eagerload_all( "dataset.actions" ) ) \ + .order_by( trans.model.HistoryDatasetAssociation.hid ) + if not show_deleted: + query = query.filter( trans.model.HistoryDatasetAssociation.deleted == False ) + if not show_purged: + query = query.filter( trans.model.Dataset.purged == False ) + return query.all() + + def get_hda_state_counts( self, trans, history, include_deleted=False, include_hidden=False ): + """ + Returns a dictionary with state counts for history's HDAs. Key is a + dataset state, value is the number of states in that count. + """ + # Build query to get (state, count) pairs. + cols_to_select = [ trans.app.model.Dataset.table.c.state, func.count( '*' ) ] + from_obj = trans.app.model.HistoryDatasetAssociation.table.join( trans.app.model.Dataset.table ) + + conditions = [ trans.app.model.HistoryDatasetAssociation.table.c.history_id == history.id ] + if not include_deleted: + # Only count datasets that have not been deleted. + conditions.append( trans.app.model.HistoryDatasetAssociation.table.c.deleted == False ) + if not include_hidden: + # Only count datasets that are visible. + conditions.append( trans.app.model.HistoryDatasetAssociation.table.c.visible == True ) + + group_by = trans.app.model.Dataset.table.c.state + query = select( columns=cols_to_select, + from_obj=from_obj, + whereclause=and_( *conditions ), + group_by=group_by ) + + # Initialize count dict with all states. + state_count_dict = {} + for k, state in trans.app.model.Dataset.states.items(): + state_count_dict[ state ] = 0 + + # Process query results, adding to count dict. + for row in trans.sa_session.execute( query ): + state, count = row + state_count_dict[ state ] = count + + return state_count_dict + + def get_hda_summary_dicts( self, trans, history ): + """Returns a list of dictionaries containing summary information + for each HDA in the given history. + """ + hda_model = trans.model.HistoryDatasetAssociation + + # get state, name, etc. + columns = ( hda_model.name, hda_model.hid, hda_model.id, hda_model.deleted, + trans.model.Dataset.state ) + column_keys = [ "name", "hid", "id", "deleted", "state" ] + + query = ( trans.sa_session.query( *columns ) + .enable_eagerloads( False ) + .filter( hda_model.history == history ) + .join( trans.model.Dataset ) + .order_by( hda_model.hid ) ) + + # build dictionaries, adding history id and encoding all ids + hda_dicts = [] + for hda_tuple in query.all(): + hda_dict = dict( zip( column_keys, hda_tuple ) ) + hda_dict[ 'history_id' ] = history.id + trans.security.encode_dict_ids( hda_dict ) + hda_dicts.append( hda_dict ) + return hda_dicts + + def _get_hda_state_summaries( self, trans, hda_dict_list ): + """Returns two dictionaries (in a tuple): state_counts and state_ids. + Each is keyed according to the possible hda states: + _counts contains a sum of the datasets in each state + _ids contains a list of the encoded ids for each hda in that state + + hda_dict_list should be a list of hda data in dictionary form. + """ + #TODO: doc to rst + # init counts, ids for each state + state_counts = {} + state_ids = {} + for key, state in trans.app.model.Dataset.states.items(): + state_counts[ state ] = 0 + state_ids[ state ] = [] + + for hda_dict in hda_dict_list: + item_state = hda_dict['state'] + if not hda_dict['deleted']: + state_counts[ item_state ] = state_counts[ item_state ] + 1 + # needs to return all ids (no deleted check) + state_ids[ item_state ].append( hda_dict['id'] ) + + return ( state_counts, state_ids ) + + def _get_history_state_from_hdas( self, trans, history, hda_state_counts ): + """Returns the history state based on the states of the HDAs it contains. + """ + states = trans.app.model.Dataset.states + + num_hdas = sum( hda_state_counts.values() ) + # (default to ERROR) + state = states.ERROR + if num_hdas == 0: + state = states.NEW + + else: + if( ( hda_state_counts[ states.RUNNING ] > 0 ) + or ( hda_state_counts[ states.SETTING_METADATA ] > 0 ) + or ( hda_state_counts[ states.UPLOAD ] > 0 ) ): + state = states.RUNNING + + elif hda_state_counts[ states.QUEUED ] > 0: + state = states.QUEUED + + elif( ( hda_state_counts[ states.ERROR ] > 0 ) + or ( hda_state_counts[ states.FAILED_METADATA ] > 0 ) ): + state = states.ERROR + + elif hda_state_counts[ states.OK ] == num_hdas: + state = states.OK + + return state + + def get_history_dict( self, trans, history, hda_dictionaries=None ): + """Returns history data in the form of a dictionary. + """ + history_dict = history.get_api_value( view='element', value_mapper={ 'id':trans.security.encode_id }) + + history_dict[ 'nice_size' ] = history.get_disk_size( nice_size=True ) + history_dict[ 'annotation' ] = history.get_item_annotation_str( trans.sa_session, trans.user, history ) + if not history_dict[ 'annotation' ]: + history_dict[ 'annotation' ] = '' + #TODO: item_slug url + + hda_summaries = hda_dictionaries if hda_dictionaries else self.get_hda_summary_dicts( trans, history ) + #TODO remove the following in v2 + ( state_counts, state_ids ) = self._get_hda_state_summaries( trans, hda_summaries ) + history_dict[ 'state_details' ] = state_counts + history_dict[ 'state_ids' ] = state_ids + history_dict[ 'state' ] = self._get_history_state_from_hdas( trans, history, state_counts ) + + return history_dict + + def set_history_from_dict( self, trans, history, new_data ): + """ + Changes history data using the given dictionary new_data. + """ + # precondition: access of the history has already been checked + + # send what we can down into the model + changed = history.set_from_dict( new_data ) + # the rest (often involving the trans) - do here + if 'annotation' in new_data.keys() and trans.get_user(): + history.add_item_annotation( trans.sa_session, trans.get_user(), history, new_data[ 'annotation' ] ) + changed[ 'annotation' ] = new_data[ 'annotation' ] + # tags + # importable (ctrl.history.set_accessible_async) + # sharing/permissions? + # slugs? + # purged - duh duh duhhhhhhnnnnnnnnnn + + if changed.keys(): + trans.sa_session.flush() + + return changed + + class UsesHistoryDatasetAssociationMixin: """ Mixin for controllers that use HistoryDatasetAssociation objects. """ @@ -817,165 +998,6 @@ step.input_connections_by_name = dict( ( conn.input_name, conn ) for conn in step.input_connections ) -class UsesHistoryMixin( SharableItemSecurityMixin ): - """ Mixin for controllers that use History objects. """ - - def get_history( self, trans, id, check_ownership=True, check_accessible=False, deleted=None ): - """Get a History from the database by id, verifying ownership.""" - history = self.get_object( trans, id, 'History', check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted ) - return self.security_check( trans, history, check_ownership, check_accessible ) - - def get_history_datasets( self, trans, history, show_deleted=False, show_hidden=False, show_purged=False ): - """ Returns history's datasets. """ - query = trans.sa_session.query( trans.model.HistoryDatasetAssociation ) \ - .filter( trans.model.HistoryDatasetAssociation.history == history ) \ - .options( eagerload( "children" ) ) \ - .join( "dataset" ) \ - .options( eagerload_all( "dataset.actions" ) ) \ - .order_by( trans.model.HistoryDatasetAssociation.hid ) - if not show_deleted: - query = query.filter( trans.model.HistoryDatasetAssociation.deleted == False ) - if not show_purged: - query = query.filter( trans.model.Dataset.purged == False ) - return query.all() - - def get_hda_state_counts( self, trans, history, include_deleted=False, include_hidden=False ): - """ - Returns a dictionary with state counts for history's HDAs. Key is a - dataset state, value is the number of states in that count. - """ - # Build query to get (state, count) pairs. - cols_to_select = [ trans.app.model.Dataset.table.c.state, func.count( '*' ) ] - from_obj = trans.app.model.HistoryDatasetAssociation.table.join( trans.app.model.Dataset.table ) - - conditions = [ trans.app.model.HistoryDatasetAssociation.table.c.history_id == history.id ] - if not include_deleted: - # Only count datasets that have not been deleted. - conditions.append( trans.app.model.HistoryDatasetAssociation.table.c.deleted == False ) - if not include_hidden: - # Only count datasets that are visible. - conditions.append( trans.app.model.HistoryDatasetAssociation.table.c.visible == True ) - - group_by = trans.app.model.Dataset.table.c.state - query = select( columns=cols_to_select, - from_obj=from_obj, - whereclause=and_( *conditions ), - group_by=group_by ) - - # Initialize count dict with all states. - state_count_dict = {} - for k, state in trans.app.model.Dataset.states.items(): - state_count_dict[ state ] = 0 - - # Process query results, adding to count dict. - for row in trans.sa_session.execute( query ): - state, count = row - state_count_dict[ state ] = count - - return state_count_dict - - def get_hda_summary_dicts( self, trans, history ): - """Returns a list of dictionaries containing summary information - for each HDA in the given history. - """ - hda_model = trans.model.HistoryDatasetAssociation - - # get state, name, etc. - columns = ( hda_model.name, hda_model.hid, hda_model.id, hda_model.deleted, - trans.model.Dataset.state ) - column_keys = [ "name", "hid", "id", "deleted", "state" ] - - query = ( trans.sa_session.query( *columns ) - .enable_eagerloads( False ) - .filter( hda_model.history == history ) - .join( trans.model.Dataset ) - .order_by( hda_model.hid ) ) - - # build dictionaries, adding history id and encoding all ids - hda_dicts = [] - for hda_tuple in query.all(): - hda_dict = dict( zip( column_keys, hda_tuple ) ) - hda_dict[ 'history_id' ] = history.id - trans.security.encode_dict_ids( hda_dict ) - hda_dicts.append( hda_dict ) - return hda_dicts - - def _get_hda_state_summaries( self, trans, hda_dict_list ): - """Returns two dictionaries (in a tuple): state_counts and state_ids. - Each is keyed according to the possible hda states: - _counts contains a sum of the datasets in each state - _ids contains a list of the encoded ids for each hda in that state - - hda_dict_list should be a list of hda data in dictionary form. - """ - #TODO: doc to rst - # init counts, ids for each state - state_counts = {} - state_ids = {} - for key, state in trans.app.model.Dataset.states.items(): - state_counts[ state ] = 0 - state_ids[ state ] = [] - - for hda_dict in hda_dict_list: - item_state = hda_dict['state'] - if not hda_dict['deleted']: - state_counts[ item_state ] = state_counts[ item_state ] + 1 - # needs to return all ids (no deleted check) - state_ids[ item_state ].append( hda_dict['id'] ) - - return ( state_counts, state_ids ) - - def _get_history_state_from_hdas( self, trans, history, hda_state_counts ): - """Returns the history state based on the states of the HDAs it contains. - """ - states = trans.app.model.Dataset.states - - num_hdas = sum( hda_state_counts.values() ) - # (default to ERROR) - state = states.ERROR - if num_hdas == 0: - state = states.NEW - - else: - if( ( hda_state_counts[ states.RUNNING ] > 0 ) - or ( hda_state_counts[ states.SETTING_METADATA ] > 0 ) - or ( hda_state_counts[ states.UPLOAD ] > 0 ) ): - state = states.RUNNING - - elif hda_state_counts[ states.QUEUED ] > 0: - state = states.QUEUED - - elif( ( hda_state_counts[ states.ERROR ] > 0 ) - or ( hda_state_counts[ states.FAILED_METADATA ] > 0 ) ): - state = states.ERROR - - elif hda_state_counts[ states.OK ] == num_hdas: - state = states.OK - - return state - - def get_history_dict( self, trans, history, hda_dictionaries=None ): - """Returns history data in the form of a dictionary. - """ - history_dict = history.get_api_value( view='element', value_mapper={ 'id':trans.security.encode_id }) - - history_dict[ 'nice_size' ] = history.get_disk_size( nice_size=True ) - - #TODO: separate, move to annotation api, fill on the client - history_dict[ 'annotation' ] = history.get_item_annotation_str( trans.sa_session, trans.user, history ) - if not history_dict[ 'annotation' ]: - history_dict[ 'annotation' ] = '' - - hda_summaries = hda_dictionaries if hda_dictionaries else self.get_hda_summary_dicts( trans, history ) - #TODO remove the following in v2 - ( state_counts, state_ids ) = self._get_hda_state_summaries( trans, hda_summaries ) - history_dict[ 'state_details' ] = state_counts - history_dict[ 'state_ids' ] = state_ids - history_dict[ 'state' ] = self._get_history_state_from_hdas( trans, history, state_counts ) - - return history_dict - - class UsesFormDefinitionsMixin: """Mixin for controllers that use Galaxy form objects.""" diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -1,12 +1,18 @@ """ API operations on a history. """ -import logging + +import pkg_resources +pkg_resources.require("Paste") +from paste.httpexceptions import HTTPBadRequest + from galaxy import web, util from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin from galaxy.web import url_for from galaxy.model.orm import desc +from galaxy.util.bunch import Bunch +import logging log = logging.getLogger( __name__ ) class HistoriesController( BaseAPIController, UsesHistoryMixin ): @@ -18,6 +24,7 @@ GET /api/histories/deleted Displays a collection (list) of histories. """ + #TODO: query (by name, date, etc.) rval = [] deleted = util.string_as_bool( deleted ) try: @@ -50,6 +57,8 @@ GET /api/histories/most_recently_used Displays information about a history. """ + #TODO: GET /api/histories/{encoded_history_id}?as_archive=True + #TODO: GET /api/histories/s/{username}/{slug} history_id = id deleted = util.string_as_bool( deleted ) @@ -92,6 +101,10 @@ trans.sa_session.flush() item = new_history.get_api_value(view='element', value_mapper={'id':trans.security.encode_id}) item['url'] = url_for( 'history', id=item['id'] ) + + #TODO: copy own history + #TODO: import an importable history + #TODO: import from archive return item @web.expose_api @@ -146,3 +159,66 @@ trans.sa_session.add( history ) trans.sa_session.flush() return 'OK' + + @web.expose_api + def update( self, trans, id, payload, **kwd ): + """ + PUT /api/histories/{encoded_history_id} + Changes an existing history. + """ + #TODO: PUT /api/histories/{encoded_history_id} payload = { rating: rating } (w/ no security checks) + try: + history = self.get_history( trans, id, check_ownership=True, check_accessible=True, deleted=True ) + # validation handled here and some parsing, processing, and conversion + payload = self._validate_and_parse_update_payload( payload ) + # additional checks here (security, etc.) + changed = self.set_history_from_dict( trans, history, payload ) + + except Exception, exception: + log.error( 'Update of history (%s) failed: %s', id, str( exception ), exc_info=True ) + # convert to appropo HTTP code + if( isinstance( exception, ValueError ) + or isinstance( exception, AttributeError ) ): + # bad syntax from the validater/parser + trans.response.status = 400 + else: + trans.response.status = 500 + return { 'error': str( exception ) } + + return changed + + def _validate_and_parse_update_payload( self, payload ): + """ + Validate and parse incomming data payload for a history. + """ + # This layer handles (most of the stricter idiot proofing): + # - unknown/unallowed keys + # - changing data keys from api key to attribute name + # - protection against bad data form/type + # - protection against malicious data content + # all other conversions and processing (such as permissions, etc.) should happen down the line + for key, val in payload.items(): + # TODO: lots of boilerplate here, but overhead on abstraction is equally onerous + if key == 'name': + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + raise ValueError( 'name must be a string or unicode: %s' %( str( type( val ) ) ) ) + payload[ 'name' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + #TODO:?? if sanitized != val: log.warn( 'script kiddie' ) + elif key == 'deleted': + if not isinstance( val, bool ): + raise ValueError( 'deleted must be a boolean: %s' %( str( type( val ) ) ) ) + elif key == 'published': + if not isinstance( payload[ 'published' ], bool ): + raise ValueError( 'published must be a boolean: %s' %( str( type( val ) ) ) ) + elif key == 'genome_build': + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + raise ValueError( 'genome_build must be a string: %s' %( str( type( val ) ) ) ) + payload[ 'genome_build' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + elif key == 'annotation': + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + raise ValueError( 'annotation must be a string or unicode: %s' %( str( type( val ) ) ) ) + payload[ 'annotation' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + else: + raise AttributeError( 'unknown key: %s' %( str( key ) ) ) + return payload + diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -724,10 +724,10 @@ 'include_hidden' : include_hidden, 'include_deleted' : include_deleted } history_exp_tool.execute( trans, incoming = params, set_output_hid = True ) + url = url_for( controller='history', action="export_archive", id=id, qualified=True ) return trans.show_message( "Exporting History '%(n)s'. Use this link to download \ the archive or import it to another Galaxy server: \ - <a href='%(u)s'>%(u)s</a>" \ - % ( { 'n' : history.name, 'u' : url_for(controller='history', action="export_archive", id=id, qualified=True ) } ) ) + <a href='%(u)s'>%(u)s</a>" % ( { 'n' : history.name, 'u' : url } ) ) @web.expose @web.json @@ -739,7 +739,8 @@ trans.sa_session.flush() return_dict = { "name" : history.name, - "link" : url_for(controller='history', action="display_by_username_and_slug", username=history.user.username, slug=history.slug ) } + "link" : url_for(controller='history', action="display_by_username_and_slug", + username=history.user.username, slug=history.slug ) } return return_dict @web.expose diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -167,7 +167,8 @@ history_dictionary = self.get_history_dict( trans, history, hda_dictionaries=hda_dictionaries ) except Exception, exc: - log.error( 'Error bootstrapping history for user %d: %s', trans.user.id, str( exc ), exc_info=True ) + user_id = str( trans.user.id ) if trans.user else '(anonymous)' + log.error( 'Error bootstrapping history for user %s: %s', user_id, str( exc ), exc_info=True ) message, status = err_msg() history_dictionary[ 'error' ] = message diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 test/casperjs/api-history-tests.js --- /dev/null +++ b/test/casperjs/api-history-tests.js @@ -0,0 +1,372 @@ +/* Utility to load a specific page and output html, page text, or a screenshot + * Optionally wait for some time, text, or dom selector + */ +try { + //...if there's a better way - please let me know, universe + var scriptDir = require( 'system' ).args[3] + // remove the script filename + .replace( /[\w|\.|\-|_]*$/, '' ) + // if given rel. path, prepend the curr dir + .replace( /^(?!\/)/, './' ), + spaceghost = require( scriptDir + 'spaceghost' ).create({ + // script options here (can be overridden by CLI) + //verbose: true, + //logLevel: debug, + scriptDir: scriptDir + }); + +} catch( error ){ + console.debug( error ); + phantom.exit( 1 ); +} +spaceghost.start(); + + +// =================================================================== SET UP +var utils = require( 'utils' ); + +var email = spaceghost.user.getRandomEmail(), + password = '123456'; +if( spaceghost.fixtureData.testUser ){ + email = spaceghost.fixtureData.testUser.email; + password = spaceghost.fixtureData.testUser.password; +} +spaceghost.user.loginOrRegisterUser( email, password ); + +function hasKeys( object, keysArray ){ + if( !utils.isObject( object ) ){ return false; } + for( var i=0; i<keysArray.length; i += 1 ){ + if( !object.hasOwnProperty( keysArray[i] ) ){ return false; } + } + return true; +} + +function countKeys( object ){ + if( !utils.isObject( object ) ){ return 0; } + var count = 0; + for( var key in object ){ + if( object.hasOwnProperty( key ) ){ count += 1; } + } + return count; +} + +// =================================================================== TESTS +spaceghost.thenOpen( spaceghost.baseUrl ).then( function(){ + + // ------------------------------------------------------------------------------------------- INDEX + this.test.comment( 'index should get a list of histories' ); + var historyIndex = this.api.histories.index(); + //this.debug( this.jsonStr( historyIndex ) ); + this.test.assert( utils.isArray( historyIndex ), "index returned an array: length " + historyIndex.length ); + this.test.assert( historyIndex.length >= 1, 'Has at least one history' ); + + var firstHistory = historyIndex[0]; + this.test.assert( hasKeys( firstHistory, [ 'id', 'name', 'url' ] ), 'Has the proper keys' ); + this.test.assert( this.api.isEncodedId( firstHistory.id ), 'Id appears well-formed' ); + + + // ------------------------------------------------------------------------------------------- SHOW + this.test.comment( 'show should get a history details object' ); + var historyShow = this.api.histories.show( firstHistory.id ); + //this.debug( this.jsonStr( historyShow ) ); + this.test.assert( hasKeys( historyShow, [ + 'id', 'name', 'annotation', 'nice_size', 'contents_url', + 'state', 'state_details', 'state_ids' ]), + 'Has the proper keys' ); + + this.test.comment( 'a history details object should contain two objects named state_details and state_ids' ); + var states = [ + 'discarded', 'empty', 'error', 'failed_metadata', 'new', + 'ok', 'paused', 'queued', 'running', 'setting_metadata', 'upload' ], + state_details = historyShow.state_details, + state_ids = historyShow.state_ids; + this.test.assert( hasKeys( state_details, states ), 'state_details has the proper keys' ); + this.test.assert( hasKeys( state_ids, states ), 'state_ids has the proper keys' ); + var state_detailsAreNumbers = true; + state_idsAreArrays = true; + states.forEach( function( state ){ + if( !utils.isArray( state_ids[ state ] ) ){ state_idsAreArrays = false; } + if( !utils.isNumber( state_details[ state ] ) ){ state_detailsAreNumbers = false; } + }); + this.test.assert( state_idsAreArrays, 'state_ids values are arrays' ); + this.test.assert( state_detailsAreNumbers, 'state_details values are numbers' ); + + this.test.comment( 'calling show with "most_recently_used" should return the first history' ); + historyShow = this.api.histories.show( 'most_recently_used' ); + //this.debug( this.jsonStr( historyShow ) ); + this.test.assert( historyShow.id === firstHistory.id, 'Is the first history' ); + + this.test.comment( 'Should be able to combine calls' ); + this.test.assert( this.api.histories.show( this.api.histories.index()[0].id ).id === firstHistory.id, + 'combining function calls works' ); + + // test server bad id protection + this.test.comment( 'A bad id to show should throw an error' ); + this.assertRaises( function(){ + this.api.histories.show( '1234123412341234' ); + }, 'Error in history API at showing history detail: 400 Bad Request', 'Raises an exception' ); + + + // ------------------------------------------------------------------------------------------- CREATE + this.test.comment( 'Calling create should create a new history and allow setting the name' ); + var newHistoryName = 'Created History', + createdHistory = this.api.histories.create({ name: newHistoryName }); + //this.debug( 'returned from create:\n' + this.jsonStr( createdHistory ) ); + this.test.assert( createdHistory.name === newHistoryName, + "Name of created history (from create) is correct: " + createdHistory.name ); + + // check the index + var newFirstHistory = this.api.histories.index()[0]; + //this.debug( 'newFirstHistory:\n' + this.jsonStr( newFirstHistory ) ); + this.test.assert( newFirstHistory.name === newHistoryName, + "Name of last history (from index) is correct: " + newFirstHistory.name ); + this.test.assert( newFirstHistory.id === createdHistory.id, + "Id of last history (from index) is correct: " + newFirstHistory.id ); + + + // ------------------------------------------------------------------------------------------- DELETE + this.test.comment( 'calling delete should delete the given history and remove it from the standard index' ); + var deletedHistory = this.api.histories.delete_( createdHistory.id ); + //this.debug( 'returned from delete:\n' + this.jsonStr( deletedHistory ) ); + this.test.assert( deletedHistory === 'OK', + "Deletion returned 'OK' - even though that's not a great, informative response: " + deletedHistory ); + + newFirstHistory = this.api.histories.index()[0]; + //this.debug( 'newFirstHistory:\n' + this.jsonStr( newFirstHistory ) ); + this.test.assert( newFirstHistory.id !== createdHistory.id, + "Id of last history (from index) DOES NOT appear: " + newFirstHistory.id ); + + this.test.comment( 'calling index with delete=true should include the deleted history' ); + newFirstHistory = this.api.histories.index( true )[0]; + //this.debug( 'newFirstHistory:\n' + this.jsonStr( newFirstHistory ) ); + this.test.assert( newFirstHistory.id === createdHistory.id, + "Id of last history (from index) DOES appear using index( deleted=true ): " + newFirstHistory.id ); + + + // ------------------------------------------------------------------------------------------- UNDELETE + this.test.comment( 'calling undelete should undelete the given history and re-include it in index' ); + var undeletedHistory = this.api.histories.undelete( createdHistory.id ); + //this.debug( 'returned from undelete:\n' + this.jsonStr( undeletedHistory ) ); + this.test.assert( undeletedHistory === 'OK', + "Undeletion returned 'OK' - even though that's not a great, informative response: " + undeletedHistory ); + + newFirstHistory = this.api.histories.index()[0]; + this.debug( 'newFirstHistory:\n' + this.jsonStr( newFirstHistory ) ); + this.test.assert( newFirstHistory.id === createdHistory.id, + "Id of last history (from index) DOES appear after undeletion: " + newFirstHistory.id ); + + + //TODO: show, deleted flag + //TODO: delete, purge flag + // ------------------------------------------------------------------------------------------- UPDATE + // ........................................................................................... idiot proofing + this.test.comment( 'updating to the current value should return no value (no change)' ); + historyShow = this.api.histories.show( newFirstHistory.id ); + var returned = this.api.histories.update( newFirstHistory.id, { + name : historyShow.name + }); + this.test.assert( countKeys( returned ) === 0, "No changed returned: " + this.jsonStr( returned ) ); + + this.test.comment( 'updating using a nonsense key should fail with an error' ); + var err = {}; + try { + returned = this.api.histories.update( newFirstHistory.id, { + konamiCode : 'uuddlrlrba' + }); + } catch( error ){ + err = error; + //this.debug( this.jsonStr( err ) ); + } + this.test.assert( !!err.message, "Error occurred: " + err.message ); + this.test.assert( err.status === 400, "Error status is 400: " + err.status ); + + this.test.comment( 'updating by attempting to change type should cause an error' ); + err = {}; + try { + returned = this.api.histories.update( newFirstHistory.id, { + //name : false + deleted : 'sure why not' + }); + } catch( error ){ + err = error; + //this.debug( this.jsonStr( err ) ); + } + this.test.assert( !!err.message, "Error occurred: " + err.message ); + this.test.assert( err.status === 400, "Error status is 400: " + err.status ); + //TODO??: other type checks? + + + // ........................................................................................... name + this.test.comment( 'update should allow changing the name' ); + returned = this.api.histories.update( newFirstHistory.id, { + name : 'New name' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.name === 'New name', "Name successfully set via update: " + historyShow.name ); + + this.test.comment( 'update should sanitize any new name' ); + returned = this.api.histories.update( newFirstHistory.id, { + name : 'New name<script type="text/javascript" src="bler">alert("blah");</script>' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.name === 'New name', "Update sanitized name: " + historyShow.name ); + + //NOTE!: this fails on sqlite3 (with default setup) + try { + this.test.comment( 'update should allow unicode in names' ); + var unicodeName = '桜ゲノム'; + returned = this.api.histories.update( newFirstHistory.id, { + name : unicodeName + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.name === unicodeName, "Update accepted unicode name: " + historyShow.name ); + } catch( err ){ + //this.debug( this.jsonStr( err ) ); + if( ( err instanceof this.api.APIError ) + && ( err.status === 500 ) + && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ + this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); + } + } + + this.test.comment( 'update should allow escaped quotations in names' ); + var quotedName = '"Bler"'; + returned = this.api.histories.update( newFirstHistory.id, { + name : quotedName + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.name === quotedName, + "Update accepted escaped quotations in name: " + historyShow.name ); + + + // ........................................................................................... deleted + this.test.comment( 'update should allow changing the deleted flag' ); + returned = this.api.histories.update( newFirstHistory.id, { + deleted: true + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.deleted === true, "Update set the deleted flag: " + historyShow.deleted ); + + this.test.comment( 'update should allow changing the deleted flag back' ); + returned = this.api.histories.update( newFirstHistory.id, { + deleted: false + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.deleted === false, "Update set the deleted flag: " + historyShow.deleted ); + + + // ........................................................................................... published + this.test.comment( 'update should allow changing the published flag' ); + returned = this.api.histories.update( newFirstHistory.id, { + published: true + }); + this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.published === true, "Update set the published flag: " + historyShow.published ); + + + // ........................................................................................... genome_build + this.test.comment( 'update should allow changing the genome_build' ); + returned = this.api.histories.update( newFirstHistory.id, { + genome_build : 'hg18' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.genome_build === 'hg18', + "genome_build successfully set via update: " + historyShow.genome_build ); + + this.test.comment( 'update should sanitize any genome_build' ); + returned = this.api.histories.update( newFirstHistory.id, { + genome_build : 'hg18<script type="text/javascript" src="bler">alert("blah");</script>' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.genome_build === 'hg18', + "Update sanitized genome_build: " + historyShow.genome_build ); + + this.test.comment( 'update should allow unicode in genome builds' ); + var unicodeBuild = '桜12'; + //NOTE!: this fails on sqlite3 (with default setup) + try { + returned = this.api.histories.update( newFirstHistory.id, { + name : unicodeBuild + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.genome_build === unicodeBuild, + "Update accepted unicode genome_build: " + historyShow.name ); + } catch( err ){ + //this.debug( this.jsonStr( err ) ); + if( ( err instanceof this.api.APIError ) + && ( err.status === 500 ) + && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ + this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); + } + } + + + // ........................................................................................... annotation + this.test.comment( 'update should allow changing the annotation' ); + var newAnnotation = 'Here are some notes that I stole from the person next to me'; + returned = this.api.histories.update( newFirstHistory.id, { + annotation : newAnnotation + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.annotation === newAnnotation, + "Annotation successfully set via update: " + historyShow.annotation ); + + this.test.comment( 'update should sanitize any new annotation' ); + returned = this.api.histories.update( newFirstHistory.id, { + annotation : 'New annotation<script type="text/javascript" src="bler">alert("blah");</script>' + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.annotation === 'New annotation', + "Update sanitized annotation: " + historyShow.annotation ); + + //NOTE!: this fails on sqlite3 (with default setup) + try { + this.test.comment( 'update should allow unicode in annotations' ); + var unicodeAnnotation = 'お願いは、それが落下させない'; + returned = this.api.histories.update( newFirstHistory.id, { + annotation : unicodeAnnotation + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.annotation === unicodeAnnotation, + "Update accepted unicode annotation: " + historyShow.annotation ); + } catch( err ){ + //this.debug( this.jsonStr( err ) ); + if( ( err instanceof this.api.APIError ) + && ( err.status === 500 ) + && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ + this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); + } + } + + this.test.comment( 'update should allow escaped quotations in annotations' ); + var quotedAnnotation = '"Bler"'; + returned = this.api.histories.update( newFirstHistory.id, { + annotation : quotedAnnotation + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.annotation === quotedAnnotation, + "Update accepted escaped quotations in annotation: " + historyShow.annotation ); + + +/* +*/ + //this.debug( this.jsonStr( historyShow ) ); +}); + +// =================================================================== +spaceghost.run( function(){ +}); diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 test/casperjs/casperjs_runner.py --- a/test/casperjs/casperjs_runner.py +++ b/test/casperjs/casperjs_runner.py @@ -361,6 +361,14 @@ self.run_js_script( 'hda-state-tests.js' ) +class Test_05_API( CasperJSTestCase ): + """Tests for API functionality and security. + """ + def test_00_history_api( self ): + """Test history API. + """ + self.run_js_script( 'api-history-tests.js' ) + # ==================================================================== MAIN if __name__ == '__main__': diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 test/casperjs/modules/api.js --- a/test/casperjs/modules/api.js +++ b/test/casperjs/modules/api.js @@ -32,11 +32,13 @@ APIError.prototype = new Error(); APIError.prototype.constructor = Error; /** @class Thrown when Galaxy the API returns an error from a request */ -function APIError( msg ){ +function APIError( msg, status ){ Error.apply( this, arguments ); this.name = "APIError"; this.message = msg; + this.status = status; } +API.prototype.APIError = APIError; exports.APIError = APIError; /* ------------------------------------------------------------------- TODO: @@ -65,7 +67,8 @@ if( resp.status !== 200 ){ // grrr... this doesn't lose the \n\r\t - throw new APIError( resp.responseText.replace( /[\s\n\r\t]+/gm, ' ' ).replace( /"/, '' ) ); + //throw new APIError( resp.responseText.replace( /[\s\n\r\t]+/gm, ' ' ).replace( /"/, '' ) ); + throw new APIError( resp.responseText, resp.status ); } return JSON.parse( resp.responseText ); }; @@ -130,7 +133,8 @@ show : 'api/histories/%s', create : 'api/histories', delete_ : 'api/histories/%s', - undelete: 'api/histories/deleted/%s/undelete' + undelete: 'api/histories/deleted/%s/undelete', + update : 'api/histories/%s' }; HistoriesAPI.prototype.index = function index( deleted ){ @@ -183,6 +187,20 @@ }); }; +HistoriesAPI.prototype.update = function create( id, payload ){ + this.api.spaceghost.info( 'history.update: ' + id + ',' + this.api.spaceghost.jsonStr( payload ) ); + + // py.payload <-> ajax.data + id = this.api.ensureId( id ); + payload = this.api.ensureObject( payload ); + url = utils.format( this.urlTpls.update, id ); + + return this.api._ajax( url, { + type : 'PUT', + data : payload + }); +}; + // =================================================================== HDAS var HDAAPI = function HDAAPI( api ){ @@ -201,7 +219,7 @@ }; HDAAPI.prototype.index = function index( historyId, ids ){ - this.api.spaceghost.info( 'history.index: ' + [ historyId, ids ] ); + this.api.spaceghost.info( 'hda.index: ' + [ historyId, ids ] ); var data = {}; if( ids ){ ids = ( utils.isArray( ids ) )?( ids.join( ',' ) ):( ids ); @@ -214,7 +232,7 @@ }; HDAAPI.prototype.show = function show( historyId, id, deleted ){ - this.api.spaceghost.info( 'history.show: ' + [ id, (( deleted )?( 'w deleted' ):( '' )) ] ); + this.api.spaceghost.info( 'hda.show: ' + [ id, (( deleted )?( 'w deleted' ):( '' )) ] ); id = ( id === 'most_recently_used' )?( id ):( this.api.ensureId( id ) ); deleted = deleted || false; @@ -224,7 +242,7 @@ }; HDAAPI.prototype.create = function create( historyId, payload ){ - this.api.spaceghost.info( 'history.create: ' + this.api.spaceghost.jsonStr( payload ) ); + this.api.spaceghost.info( 'hda.create: ' + this.api.spaceghost.jsonStr( payload ) ); // py.payload <-> ajax.data payload = this.api.ensureObject( payload ); @@ -235,7 +253,8 @@ }; HDAAPI.prototype.update = function create( historyId, id, payload ){ - this.api.spaceghost.info( 'history.update: ' + this.api.spaceghost.jsonStr( payload ) ); + this.api.spaceghost.info( 'hda.update: ' + historyId + ',' + id + ',' + + this.api.spaceghost.jsonStr( payload ) ); // py.payload <-> ajax.data historyId = this.api.ensureId( historyId ); diff -r 7c59121055516595937b83d51eaf98b60723b622 -r fb28ceb83c379e1d792f622f9b2cbc8c3e050f37 test/casperjs/spaceghost.js --- a/test/casperjs/spaceghost.js +++ b/test/casperjs/spaceghost.js @@ -545,9 +545,10 @@ * NOTE: uses string indexOf - doesn't play well with urls like [ 'history', 'history/bler' ] * @param {String} urlToWaitFor the url to wait for (rel. to spaceghost.baseUrl) * @param {Function} then the function to call after the nav request + * @param {Function} timeoutFn the function to call on timeout (optional) */ -SpaceGhost.prototype.waitForNavigation = function waitForNavigation( urlToWaitFor, then ){ - return this.waitForMultipleNavigation( [ urlToWaitFor ], then ); +SpaceGhost.prototype.waitForNavigation = function waitForNavigation( urlToWaitFor, then, timeoutFn ){ + return this.waitForMultipleNavigation( [ urlToWaitFor ], then, timeoutFn ); }; /** Wait for a multiple navigation requests then call a function. @@ -555,8 +556,9 @@ * NOTE: uses string indexOf - doesn't play well with urls like [ 'history', 'history/bler' ] * @param {String[]} urlsToWaitFor the relative urls to wait for * @param {Function} then the function to call after the nav request + * @param {Function} timeoutFn the function to call on timeout (optional) */ -SpaceGhost.prototype.waitForMultipleNavigation = function waitForMultipleNavigation( urlsToWaitFor, then ){ +SpaceGhost.prototype.waitForMultipleNavigation = function waitForMultipleNavigation( urlsToWaitFor, then, timeoutFn ){ this.info( 'waiting for navigation: ' + this.jsonStr( urlsToWaitFor ) ); function urlMatches( urlToMatch, url ){ return ( url.indexOf( spaceghost.baseUrl + '/' + urlToMatch ) !== -1 ); @@ -586,6 +588,9 @@ function callThen(){ if( utils.isFunction( then ) ){ then.call( this ); } }, + function timeout(){ + if( utils.isFunction( timeoutFn ) ){ timeoutFn.call( this ); } + }, this.options.waitTimeout * urlsToWaitFor.length ); return this; @@ -731,8 +736,9 @@ /** Casper has an (undocumented?) skip test feature. This is a conv. wrapper for that. */ -SpaceGhost.prototype.skipTest = function skipTest(){ - throw this.test.SKIP_MESSAGE; +SpaceGhost.prototype.skipTest = function skipTest( msg ){ + this.warn( 'Skipping test. ' + msg ); + //throw this.test.SKIP_MESSAGE; }; /** Test helper - within frame, assert selector, and assert text in selector Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Functional tests to verify correct setting of metadata when dependency definitions are deleted.
by commits-noreply＠bitbucket.org 23 Apr '13

23 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7c5912105551/ Changeset: 7c5912105551 User: inithello Date: 2013-04-23 20:46:53 Summary: Functional tests to verify correct setting of metadata when dependency definitions are deleted. Affected #: 1 file diff -r 1e3d92ec22b2794bfa2294edb74695c580791406 -r 7c59121055516595937b83d51eaf98b60723b622 test/tool_shed/functional/test_0440_deleting_dependency_definitions.py --- /dev/null +++ b/test/tool_shed/functional/test_0440_deleting_dependency_definitions.py @@ -0,0 +1,365 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +column_repository_name = 'column_maker_0440' +column_repository_description = "Add column" +column_repository_long_description = "Compute an expression on every row" + +convert_repository_name = 'convert_chars_0440' +convert_repository_description = "Convert delimiters" +convert_repository_long_description = "Convert delimiters to tab" + +bwa_package_repository_name = 'bwa_package_0440' +bwa_package_repository_description = "BWA Package Repository" +bwa_package_repository_long_description = "BWA repository with a package tool dependency defined for BWA 0.5.9." + +bwa_base_repository_name = 'bwa_base_0440' +bwa_base_repository_description = "BWA Base" +bwa_base_repository_long_description = "NT space mapping with BWA" + +bwa_tool_dependency_repository_name = 'bwa_tool_dependency_0440' +bwa_tool_dependency_repository_description = "BWA Base" +bwa_tool_dependency_repository_long_description = "NT space mapping with BWA" + +''' +Simple repository dependencies: +1. Create and populate column_maker_0440 so that it has an installable revision 0. +2. Create and populate convert_chars_0440 so that it has an installable revision 0. +3. Add a valid simple repository_dependencies.xml to convert_chars_0440 that points to the installable revision of column_maker_0440. +4. Make sure the installable revision of convert_chars_0440 is now revision 1 instead of revision 0. +5. Delete repository_dependencies.xml from convert_chars_0440, and make sure convert_chars_0440 now has two installable revisions: 1 and 2 + +Complex repository dependencies: +1. Create and populate bwa_package_0440 so that it has a valid orphan tool dependency definition and an installable revision 0. +2. Create and populate bwa_base_0440 so that it has an installable revision 0. +3. Add a valid complex repository dependency tool_dependencies.xml to bwa_base_0440 that points to the installable revision 0 of bwa_package_0440. +4. Make sure that bwa_base_0440 installable revision is now revision 1 instead of revision 0. +5. Delete tool_dependencies.xml from bwa_base_0440, and make sure bwa_base_0440 now has two installable revisions: 1 and 2 + +Tool dependencies: +1. Create and populate bwa_tool_dependency_0440 so that it has a valid orphan tool dependency definition and an installable revision 0. +2. Delete tool_dependencies.xml from bwa_tool_dependency_0440, and make sure that bwa_tool_dependency_0440 still has + a single installable revision 0. +3. Add the same tool_dependencies.xml file to bwa_tool_dependency_0440, and make sure that bwa_tool_dependency_0440 + still has a single installable revision 0. +''' + + +class TestDeletedDependencies( ShedTwillTestCase ): + '''Test metadata setting when dependency definitions are deleted.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts and login as an admin user.""" + """ + Create all the user accounts that are needed for this test script to run independently of other tests. + Previously created accounts will not be re-created. + """ + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + + def test_0005_create_column_maker_repository( self ): + '''Create and populate a repository named column_maker_0440.''' + ''' + We are at simple repository dependencies, step 1 - Create and populate column_maker_0440 so that it has an installable revision 0. + ''' + category = self.create_category( name='Test 0440 Deleted Dependency Definitions', + description='Description of Deleted Dependency Definitions category for test 0440' ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + strings_displayed = [ "Repository 'column_maker_0440' has been created" ] + repository = self.get_or_create_repository( name=column_repository_name, + description=column_repository_description, + long_description=column_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=strings_displayed ) + self.upload_file( repository, + filename='column_maker/column_maker.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded column maker tool tarball.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0010_create_convert_chars_repository( self ): + '''Create and populate a repository named convert_chars_0440.''' + ''' + We are at simple repository dependencies, step 2 - Create and populate convert_chars_0440 so that it has an installable revision 0. + ''' + category = test_db_util.get_category_by_name( 'Test 0440 Deleted Dependency Definitions' ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + strings_displayed = [ "Repository 'convert_chars_0440' has been created" ] + repository = self.get_or_create_repository( name=convert_repository_name, + description=convert_repository_description, + long_description=convert_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=strings_displayed ) + self.upload_file( repository, + filename='convert_chars/convert_chars.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded convert chars tool tarball.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0015_create_dependency_on_convert_chars( self ): + '''Create a dependency definition file that specifies column_maker_0440 and upload it to convert_chars_0440.''' + ''' + We are at simple repository dependencies, step 3 - Add a valid simple repository_dependencies.xml to + convert_chars_0440 that points to the installable revision of column_maker_0440. + ''' + convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + dependency_xml_path = self.generate_temp_path( 'test_0440', additional_paths=[ 'dependencies' ] ) + column_tuple = ( self.url, column_repository.name, column_repository.user.username, self.get_repository_tip( column_repository ) ) + # After this, convert_chars_0440 should depend on column_maker_0440. + self.create_repository_dependency( repository=convert_repository, + repository_tuples=[ column_tuple ], + filepath=dependency_xml_path, + prior_installation_required=True ) + self.check_repository_dependency( convert_repository, column_repository ) + + def test_0020_verify_dependency_metadata( self ): + '''Verify that uploading the dependency moved metadata to the tip.''' + ''' + We are at simple repository dependencies, step 4 - Make sure the installable revision of convert_chars_0440 is now + revision 1 (the tip) instead of revision 0. + ''' + repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + tip = self.get_repository_tip( repository ) + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, tip ) + # Make sure that the new tip is now downloadable, and that there are no other downloadable revisions. + assert metadata_record.downloadable, 'Tip is not downloadable.' + assert len( repository.downloadable_revisions ) == 1, 'Repository %s has %d downloadable revisions, expected 1.' % \ + ( repository.name, len( repository.downloadable_revisions ) ) + + def test_0025_delete_repository_dependency( self ): + '''Delete the repository_dependencies.xml from convert_chars_0440.''' + ''' + We are at simple repository dependencies, steps 5 and 6 - Delete repository_dependencies.xml from convert_chars_0440. + Make sure convert_chars_0440 now has two installable revisions: 1 and 2 + ''' + repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + # Record the current tip, so we can verify that it's still a downloadable revision after repository_dependencies.xml + # is deleted and a new downloadable revision is created. + old_changeset_revision = self.get_repository_tip( repository ) + self.delete_files_from_repository( repository, filenames=[ 'repository_dependencies.xml' ] ) + new_changeset_revision = self.get_repository_tip( repository ) + # Check that the old changeset revision is still downloadable. + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, old_changeset_revision ) + assert metadata_record.downloadable, 'The revision of %s that contains repository_dependencies.xml is no longer downloadable.' % \ + repository.name + # Check that the new tip is also downloadable. + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, new_changeset_revision ) + assert metadata_record.downloadable, 'The revision of %s that does not contain repository_dependencies.xml is not downloadable.' % \ + repository.name + # Explicitly reload the repository instance from the database, to avoid potential caching issues. + test_db_util.refresh( repository ) + # Verify that there are only two downloadable revisions. + assert len( repository.downloadable_revisions ) == 2, 'Repository %s has %d downloadable revisions, expected 2.' % \ + ( repository.name, len( repository.downloadable_revisions ) ) + + def test_0030_create_bwa_package_repository( self ): + '''Create and populate the bwa_package_0440 repository.''' + ''' + We are at complex repository dependencies, step 1 - Create and populate bwa_package_0440 so that it has a valid orphan + tool dependency definition and an installable revision 0. + ''' + category = test_db_util.get_category_by_name( 'Test 0440 Deleted Dependency Definitions' ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + strings_displayed = [ "Repository 'bwa_package_0440' has been created" ] + repository = self.get_or_create_repository( name=bwa_package_repository_name, + description=bwa_package_repository_description, + long_description=bwa_package_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=strings_displayed ) + self.upload_file( repository, + filename='bwa/complex/tool_dependencies.xml', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded package tool dependency definition.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0035_create_bwa_base_repository( self ): + '''Create and populate the bwa_base_0440 repository.''' + ''' + We are at complex repository dependencies, step 2 - Create and populate bwa_base_0440 so that it has an installable revision 0. + This repository should contain a tool with a defined dependency that will be satisfied by the tool dependency defined in bwa_package_0440. + ''' + category = test_db_util.get_category_by_name( 'Test 0440 Deleted Dependency Definitions' ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + strings_displayed = [ "Repository 'bwa_base_0440' has been created" ] + repository = self.get_or_create_repository( name=bwa_base_repository_name, + description=bwa_base_repository_description, + long_description=bwa_base_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=strings_displayed ) + self.upload_file( repository, + filename='bwa/complex/bwa_base.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded BWA nucleotide space mapping tool tarball.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0040_create_dependency_on_bwa_package_repository( self ): + '''Create a complex repository dependency on bwa_package_0440 and upload it to bwa_tool_0440.''' + ''' + We are at complex repository dependencies, step 3 - Add a valid complex repository dependency tool_dependencies.xml to + bwa_base_0440 that points to the installable revision 0 of bwa_package_0440. + ''' + bwa_package_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) + bwa_base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + dependency_path = self.generate_temp_path( 'test_0440', additional_paths=[ 'complex' ] ) + changeset_revision = self.get_repository_tip( bwa_package_repository ) + bwa_tuple = ( self.url, bwa_package_repository.name, bwa_package_repository.user.username, changeset_revision ) + self.create_repository_dependency( repository=bwa_base_repository, + repository_tuples=[ bwa_tuple ], + filepath=dependency_path, + prior_installation_required=True, + complex=True, + package='bwa', + version='0.5.9' ) + + def test_0045_verify_dependency_metadata( self ): + '''Verify that uploading the dependency moved metadata to the tip.''' + ''' + We are at complex repository dependencies, step 4 - Make sure that bwa_base_0440 installable revision is now revision 1 + instead of revision 0. + ''' + repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + tip = self.get_repository_tip( repository ) + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, tip ) + # Make sure that the new tip is now downloadable, and that there are no other downloadable revisions. + assert metadata_record.downloadable, 'Tip is not downloadable.' + assert len( repository.downloadable_revisions ) == 1, 'Repository %s has %d downloadable revisions, expected 1.' % \ + ( repository.name, len( repository.downloadable_revisions ) ) + + def test_0050_delete_complex_repository_dependency( self ): + '''Delete the tool_dependencies.xml from bwa_base_0440.''' + ''' + We are at complex repository dependencies, step 5 - Delete tool_dependencies.xml from bwa_base_0440, + and make sure bwa_base_0440 now has two installable revisions: 1 and 2 + ''' + repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + # Record the current tip, so we can verify that it's still a downloadable revision after tool_dependencies.xml + # is deleted and a new downloadable revision is created. + old_changeset_revision = self.get_repository_tip( repository ) + self.delete_files_from_repository( repository, filenames=[ 'tool_dependencies.xml' ] ) + new_changeset_revision = self.get_repository_tip( repository ) + # Check that the old changeset revision is still downloadable. + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, old_changeset_revision ) + assert metadata_record.downloadable, 'The revision of %s that contains tool_dependencies.xml is no longer downloadable.' % \ + repository.name + # Check that the new tip is also downloadable. + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, new_changeset_revision ) + assert metadata_record.downloadable, 'The revision of %s that does not contain tool_dependencies.xml is not downloadable.' % \ + repository.name + # Verify that there are only two downloadable revisions. + assert len( repository.downloadable_revisions ) == 2, 'Repository %s has %d downloadable revisions, expected 2.' % \ + ( repository.name, len( repository.downloadable_revisions ) ) + + def test_0055_create_bwa_tool_dependency_repository( self ): + '''Create and populate the bwa_tool_dependency_0440 repository.''' + ''' + We are at tool dependencies, step 1 - Create and populate bwa_tool_dependency_0440 so that it has a valid orphan tool + dependency definition and an installable revision 0. + ''' + category = test_db_util.get_category_by_name( 'Test 0440 Deleted Dependency Definitions' ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + strings_displayed = [ "Repository 'bwa_tool_dependency_0440' has been created" ] + repository = self.get_or_create_repository( name=bwa_tool_dependency_repository_name, + description=bwa_tool_dependency_repository_description, + long_description=bwa_tool_dependency_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=strings_displayed ) + self.upload_file( repository, + filename='bwa/complex/tool_dependencies.xml', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded package tool dependency definition.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0060_delete_bwa_tool_dependency_definition( self ): + '''Delete the tool_dependencies.xml file from bwa_tool_dependency_0440.''' + ''' + We are at tool dependencies, step 2 - Delete tool_dependencies.xml from bwa_tool_dependency_0440. + Make sure bwa_tool_dependency_0440 still has a downloadable changeset revision at the old tip. + ''' + repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_dependency_repository_name, common.test_user_1_name ) + # Record the current tip, so we can verify that it's still a downloadable revision after repository_dependencies.xml + # is deleted and a new downloadable revision is created. + old_changeset_revision = self.get_repository_tip( repository ) + self.delete_files_from_repository( repository, filenames=[ 'tool_dependencies.xml' ] ) + new_changeset_revision = self.get_repository_tip( repository ) + # Check that the old changeset revision is still downloadable. + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, old_changeset_revision ) + assert metadata_record.downloadable, 'The revision of %s that contains tool_dependencies.xml is no longer downloadable.' % \ + repository.name + # Check that the new tip does not have a metadata revision. + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, new_changeset_revision ) + # If a changeset revision does not have metadata, the above method will return None. + assert metadata_record is None, 'The tip revision of %s should not have metadata, but metadata was found.' % repository.name + # Verify that the new changeset revision is not downloadable. + assert len( repository.downloadable_revisions ) == 1, 'Repository %s has %d downloadable revisions, expected 1.' % \ + ( repository.name, len( repository.downloadable_revisions ) ) + + def test_0065_reupload_bwa_tool_dependency_definition( self ): + '''Reupload the tool_dependencies.xml file to bwa_tool_dependency_0440.''' + ''' + We are at tool dependencies, step 3 - Add the same tool_dependencies.xml file to bwa_tool_dependency_0440, and make sure + that bwa_tool_dependency_0440 still has a single installable revision 0. + ''' + repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_dependency_repository_name, common.test_user_1_name ) + # Record the current tip, so we can verify that it's still not a downloadable revision after tool_dependencies.xml + # is re-uploaded and a new downloadable revision is created. + old_changeset_revision = self.get_repository_tip( repository ) + self.upload_file( repository, + filename='bwa/complex/tool_dependencies.xml', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded package tool dependency definition.', + strings_displayed=[], + strings_not_displayed=[] ) + new_changeset_revision = self.get_repository_tip( repository ) + # Check that the old changeset revision is still downloadable. + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, old_changeset_revision ) + assert metadata_record is None, 'The revision of %s that does not contain tool_dependencies.xml should not be downloadable, but is.' % \ + repository.name + # Check that the new tip is also downloadable. + metadata_record = self.get_repository_metadata_by_changeset_revision( repository, new_changeset_revision ) + assert metadata_record.downloadable, 'The revision of %s that contains tool_dependencies.xml is not downloadable.' % \ + repository.name + # Verify that there are only two downloadable revisions. + assert len( repository.downloadable_revisions ) == 1, 'Repository %s has %d downloadable revisions, expected 1.' % \ + ( repository.name, len( repository.downloadable_revisions ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Have data source tools display the provided 'name' parameter as the initial dataset name.
by commits-noreply＠bitbucket.org 22 Apr '13

22 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1e3d92ec22b2/ Changeset: 1e3d92ec22b2 User: dan Date: 2013-04-22 22:58:49 Summary: Have data source tools display the provided 'name' parameter as the initial dataset name. Affected #: 3 files diff -r ea0b7ca55aec1552718132227d80b0ae82ca2913 -r 1e3d92ec22b2794bfa2294edb74695c580791406 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -36,6 +36,7 @@ from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy.jobs import ParallelismInfo from galaxy.tools.actions import DefaultToolAction +from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.deps import DependencyManager from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings @@ -3028,6 +3029,7 @@ allow the user to query and extract data from another web site. """ tool_type = 'data_source' + default_tool_action = DataSourceToolAction def _build_GALAXY_URL_parameter( self ): return ToolParameter.build( self, ElementTree.XML( '<param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=%s" />' % self.id ) ) diff -r ea0b7ca55aec1552718132227d80b0ae82ca2913 -r 1e3d92ec22b2794bfa2294edb74695c580791406 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -342,9 +342,10 @@ params['on_string'] = on_text data.name = fill_template( output.label, context=params ) else: - data.name = tool.name - if on_text: - data.name += ( " on " + on_text ) + if params is None: + params = make_dict_copy( incoming ) + wrap_values( tool.inputs, params, skip_missing_values = not tool.check_values ) + data.name = self._get_default_data_name( data, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params ) # Store output out_data[ name ] = data if output.actions: @@ -430,3 +431,9 @@ trans.app.job_queue.put( job.id, job.tool_id ) trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) return job, out_data + + def _get_default_data_name( self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd ): + name = tool.name + if on_text: + name += ( " on " + on_text ) + return name diff -r ea0b7ca55aec1552718132227d80b0ae82ca2913 -r 1e3d92ec22b2794bfa2294edb74695c580791406 lib/galaxy/tools/actions/data_source.py --- /dev/null +++ b/lib/galaxy/tools/actions/data_source.py @@ -0,0 +1,12 @@ +from __init__ import DefaultToolAction + +import logging +log = logging.getLogger( __name__ ) + +class DataSourceToolAction( DefaultToolAction ): + """Tool action used for Data Source Tools""" + + def _get_default_data_name( self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd ): + if incoming and 'name' in incoming: + return incoming[ 'name' ] + return super( DataSourceToolAction, self )._get_default_data_name( dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Raise exceptions when tool dependency definition <install> and <set_environment> tags define an unsupported version attribute.
by commits-noreply＠bitbucket.org 22 Apr '13

22 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ea0b7ca55aec/ Changeset: ea0b7ca55aec User: greg Date: 2013-04-22 20:51:48 Summary: Raise exceptions when tool dependency definition <install> and <set_environment> tags define an unsupported version attribute. Affected #: 1 file diff -r 42632cc2a166e8933fe29496a6087309769b8987 -r ea0b7ca55aec1552718132227d80b0ae82ca2913 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -151,6 +151,8 @@ action_dict[ env_elem.tag ] = env_var_dicts actions.append( ( action_type, action_dict ) ) return tool_dependency, actions + else: + raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) return None, actions def install_and_build_package_via_fabric( app, tool_dependency, actions_dict ): @@ -299,6 +301,8 @@ sa_session.refresh( tool_dependency ) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: print package_name, 'version', package_version, 'installed in', install_dir + else: + raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) elif package_elem.tag == 'readme': # Nothing to be done. continue @@ -550,6 +554,8 @@ sa_session.add( tool_dependency ) sa_session.flush() print 'Environment variable ', env_var_name, 'set in', install_dir + else: + raise NotImplementedError( 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' ) def strip_path( fpath ): if not fpath: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Don't use paging on tool shed repository grids since generated urls filter out needed request params when using paging.
by commits-noreply＠bitbucket.org 22 Apr '13

22 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/42632cc2a166/ Changeset: 42632cc2a166 User: greg Date: 2013-04-22 19:37:28 Summary: Don't use paging on tool shed repository grids since generated urls filter out needed request params when using paging. Affected #: 1 file diff -r 1037ab5b4f761c104b01cf0aee9222d28a67034b -r 42632cc2a166e8933fe29496a6087309769b8987 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -63,7 +63,7 @@ standard_filters = [] num_rows_per_page = 50 preserve_state = False - use_paging = True + use_paging = False class ValidCategoryGrid( CategoryGrid ): @@ -105,7 +105,7 @@ standard_filters = [] num_rows_per_page = 50 preserve_state = False - use_paging = True + use_paging = False class RepositoryGrid( grids.Grid ): @@ -310,6 +310,7 @@ allow_multiple=False, condition=( lambda item: not item.deleted ), async_compatible=False ) ] + use_paging = False def build_initial_query( self, trans, **kwd ): category_id = kwd.get( 'id', None ) @@ -353,7 +354,7 @@ default_filter = dict( deleted="False" ) num_rows_per_page = 50 preserve_state = False - use_paging = True + use_paging = False def build_initial_query( self, trans, **kwd ): decoded_user_id = trans.security.decode_id( kwd[ 'user_id' ] ) @@ -395,6 +396,7 @@ allow_multiple=False, condition=( lambda item: not item.deleted and item.deprecated ), async_compatible=False ) ] + use_paging = False def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ @@ -424,6 +426,7 @@ key="free-text-search", visible=False, filterable="standard" ) ) + use_paging = False def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ @@ -462,7 +465,7 @@ async_compatible=False ) ] global_actions = [ grids.GridAction( "User preferences", dict( controller='user', action='index', cntrller='repository' ) ) - ] + ] class MyWritableRepositoriesGrid( RepositoryGrid ): @@ -499,6 +502,7 @@ allow_multiple=False, condition=( lambda item: not item.deleted ), async_compatible=False ) ] + use_paging = False def build_initial_query( self, trans, **kwd ): # TODO: improve performance by adding a db table associating users with repositories for which they have write access. @@ -589,6 +593,7 @@ visible=False, filterable="standard" ) ) operations = [] + use_paging = False def build_initial_query( self, trans, **kwd ): if 'id' in kwd: @@ -665,7 +670,7 @@ default_filter = {} num_rows_per_page = 50 preserve_state = False - use_paging = True + use_paging = False def build_initial_query( self, trans, **kwd ): match_tuples = kwd.get( 'match_tuples', [] ) @@ -823,7 +828,7 @@ default_filter = dict( malicious="False" ) num_rows_per_page = 50 preserve_state = False - use_paging = True + use_paging = False def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.RepositoryMetadata ) \ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Tool shed functional tests for installation order with complex repository dependencies. Modified tests 0100 and 1100 to be less ambiguous regarding which repository contains what.
by commits-noreply＠bitbucket.org 22 Apr '13

22 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1037ab5b4f76/ Changeset: 1037ab5b4f76 User: inithello Date: 2013-04-22 18:53:43 Summary: Tool shed functional tests for installation order with complex repository dependencies. Modified tests 0100 and 1100 to be less ambiguous regarding which repository contains what. Affected #: 8 files diff -r a8139a367ed3de938f5d71336a06f1f768132469 -r 1037ab5b4f761c104b01cf0aee9222d28a67034b test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -800,8 +800,12 @@ self.check_for_strings( strings_displayed, strings_not_displayed ) # This section is tricky, due to the way twill handles form submission. The tool dependency checkbox needs to # be hacked in through tc.browser, putting the form field in kwd doesn't work. + form = tc.browser.get_form( 'select_tool_panel_section' ) + submit_button = 'select_tool_panel_section_button' + if form is None: + form = tc.browser.get_form( 'select_shed_tool_panel_config' ) + submit_button = 'select_shed_tool_panel_config_button' if 'install_tool_dependencies' in self.last_page(): - form = tc.browser.get_form( 'select_tool_panel_section' ) checkbox = form.find_control( id="install_tool_dependencies" ) checkbox.disabled = False if install_tool_dependencies: @@ -816,14 +820,10 @@ kwd[ 'shed_tool_conf' ] = self.shed_tool_conf if new_tool_panel_section: kwd[ 'new_tool_panel_section' ] = new_tool_panel_section - if includes_tools_for_display_in_tool_panel: - self.submit_form( 1, 'select_tool_panel_section_button', **kwd ) - self.check_for_strings( post_submit_strings_displayed, strings_not_displayed ) - else: - self.check_for_strings( strings_displayed=[ 'Choose the configuration file whose tool_path setting will be used for installing repositories' ] ) - args = dict( shed_tool_conf=self.shed_tool_conf ) - self.submit_form( 1, 'select_shed_tool_panel_config_button', **args ) - self.check_for_strings( post_submit_strings_displayed, strings_not_displayed ) + if not includes_tools_for_display_in_tool_panel: + self.check_for_strings( strings_displayed=[ 'Choose the configuration file' ] ) + self.submit_form( 1, submit_button, **kwd ) + self.check_for_strings( post_submit_strings_displayed, strings_not_displayed ) repository_ids = self.initiate_installation_process( new_tool_panel_section=new_tool_panel_section ) self.wait_for_repository_installation( repository_ids ) diff -r a8139a367ed3de938f5d71336a06f1f768132469 -r 1037ab5b4f761c104b01cf0aee9222d28a67034b test/tool_shed/functional/test_0100_complex_repository_dependencies.py --- a/test/tool_shed/functional/test_0100_complex_repository_dependencies.py +++ b/test/tool_shed/functional/test_0100_complex_repository_dependencies.py @@ -6,11 +6,11 @@ bwa_base_repository_name = 'bwa_base_repository_0100' bwa_base_repository_description = "BWA Base" -bwa_base_repository_long_description = "BWA tool that depends on bwa 0.5.9, with a complex repository dependency pointing at bwa_tool_repository_0100" +bwa_base_repository_long_description = "BWA tool that depends on bwa 0.5.9, with a complex repository dependency pointing at package_bwa_0_5_9_0100" -bwa_tool_repository_name = 'bwa_tool_repository_0100' -bwa_tool_repository_description = "BWA Tool" -bwa_tool_repository_long_description = "BWA repository with a package tool dependency defined for BWA 0.5.9." +bwa_package_repository_name = 'package_bwa_0_5_9_0100' +bwa_package_repository_description = "BWA Tool" +bwa_package_repository_long_description = "BWA repository with a package tool dependency defined for BWA 0.5.9." category_name = 'Test 0100 Complex Repository Dependencies' category_description = 'Test 0100 Complex Repository Dependencies' @@ -32,15 +32,15 @@ assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) - def test_0005_create_bwa_tool_repository( self ): - '''Create and populate bwa_tool_repository_0100.''' + def test_0005_create_bwa_package_repository( self ): + '''Create and populate package_bwa_0_5_9_0100.''' category = self.create_category( name=category_name, description=category_description ) self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - # Create a repository named bwa_tool_repository_0100 owned by user1. - repository = self.get_or_create_repository( name=bwa_tool_repository_name, - description=bwa_tool_repository_description, - long_description=bwa_tool_repository_long_description, + # Create a repository named package_bwa_0_5_9_0100 owned by user1. + repository = self.get_or_create_repository( name=bwa_package_repository_name, + description=bwa_package_repository_description, + long_description=bwa_package_repository_long_description, owner=common.test_user_1_name, category_id=self.security.encode_id( category.id ), strings_displayed=[] ) @@ -53,7 +53,7 @@ commit_message='Uploaded tool_dependencies.xml.', strings_displayed=[ 'The settings for <b>name</b>, <b>version</b> and <b>type</b> from a contained tool' ], strings_not_displayed=[] ) - # Visit the manage repository page for bwa_tool_repository_0100. + # Visit the manage repository page for package_bwa_0_5_9_0100. self.display_manage_repository_page( repository, strings_displayed=[ 'Tool dependencies', 'may not be', 'in this repository' ] ) def test_0010_create_bwa_base_repository( self ): @@ -85,10 +85,10 @@ xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The repository named bwa_base_repository_0100 is the dependent repository. base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - # The tool_repository named bwa_tool_repository_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + # The repository named package_bwa_0_5_9_0100 is the required repository. + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = 'http://http://this is not an url!' - name = 'bwa_tool_repository_0100' + name = 'package_bwa_0_5_9_0100' owner = 'user1' changeset_revision = self.get_repository_tip( tool_repository ) strings_displayed = [ 'Repository dependencies are currently supported only within the same tool shed' ] @@ -108,14 +108,14 @@ xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The base_repository named bwa_base_repository_0100 is the dependent repository. base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - # The tool_repository named bwa_tool_repository_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + # The repository named package_bwa_0_5_9_0100 is the required repository. + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url name = 'invalid_repository!?' owner = 'user1' changeset_revision = self.get_repository_tip( tool_repository ) strings_displayed = [ 'because the name is invalid' ] - # Populate the dependent base_repository named bwa_tool_repository_0100 with an invalid tool_dependencies.xml file. + # Populate the dependent base_repository named package_bwa_0_5_9_0100 with an invalid tool_dependencies.xml file. repository_tuple = ( url, name, owner, changeset_revision ) self.create_repository_dependency( repository=base_repository, filepath=dependency_path, @@ -131,10 +131,10 @@ xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The base_repository named bwa_base_repository_0100 is the dependent repository. base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - # The tool_repository named bwa_tool_repository_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + # The repository named package_bwa_0_5_9_0100 is the required repository. + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url - name = 'bwa_tool_repository_0100' + name = 'package_bwa_0_5_9_0100' owner = 'invalid_owner!?' changeset_revision = self.get_repository_tip( tool_repository ) strings_displayed = [ 'because the owner is invalid.' ] @@ -153,10 +153,10 @@ xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The base_repository named bwa_base_repository_0100 is the dependent repository. base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - # The tool_repository named bwa_tool_repository_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + # The repository named package_bwa_0_5_9_0100 is the required repository. + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url - name = 'bwa_tool_repository_0100' + name = 'package_bwa_0_5_9_0100' owner = 'user1' changeset_revision = '1234abcd' strings_displayed = [ 'because the changeset revision is invalid.' ] @@ -170,14 +170,14 @@ version='0.5.9' ) def test_0035_generate_complex_repository_dependency( self ): - '''Generate and upload a valid tool_dependencies.xml file that specifies bwa_tool_repository_0100.''' + '''Generate and upload a valid tool_dependencies.xml file that specifies package_bwa_0_5_9_0100.''' # The base_repository named bwa_base_repository_0100 is the dependent repository. base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - # The tool_repository named bwa_tool_repository_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + # The repository named package_bwa_0_5_9_0100 is the required repository. + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex' ] ) url = self.url - name = 'bwa_tool_repository_0100' + name = 'package_bwa_0_5_9_0100' owner = 'user1' changeset_revision = self.get_repository_tip( tool_repository ) repository_tuple = ( url, name, owner, changeset_revision ) @@ -194,8 +194,8 @@ '''Generate and upload a new tool_dependencies.xml file that specifies an arbitrary file on the filesystem, and verify that bwa_base depends on the new changeset revision.''' # The base_repository named bwa_base_repository_0100 is the dependent repository. base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - # The tool_repository named bwa_tool_repository_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + # The repository named package_bwa_0_5_9_0100 is the required repository. + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) previous_changeset = self.get_repository_tip( tool_repository ) old_tool_dependency = self.get_filename( os.path.join( 'bwa', 'complex', 'readme', 'tool_dependencies.xml' ) ) new_tool_dependency_path = self.generate_temp_path( 'test_1100', additional_paths=[ 'tool_dependency' ] ) @@ -220,12 +220,12 @@ self.display_manage_repository_page( tool_repository, strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) - # Visit the manage page of the bwa_tool_repository_0100 to confirm the valid tool dependency definition. + # Visit the manage page of the package_bwa_0_5_9_0100 to confirm the valid tool dependency definition. self.display_manage_repository_page( tool_repository, strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) # Visit the manage page of the bwa_base_repository_0100 to confirm the valid tool dependency definition - # and the updated changeset revision (updated tip) of the bwa_tool_repository_0100 repository is displayed + # and the updated changeset revision (updated tip) of the package_bwa_0_5_9_0100 repository is displayed # as the required repository revision. The original revision defined in the previously uploaded # tool_dependencies.xml file will be updated. self.display_manage_repository_page( base_repository, diff -r a8139a367ed3de938f5d71336a06f1f768132469 -r 1037ab5b4f761c104b01cf0aee9222d28a67034b test/tool_shed/functional/test_0170_complex_prior_installation_required.py --- /dev/null +++ b/test/tool_shed/functional/test_0170_complex_prior_installation_required.py @@ -0,0 +1,137 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +matplotlib_repository_name = 'package_matplotlib_1_2_0170' +matplotlib_repository_description = "Contains a tool dependency definition that downloads and compiles version 1.2.x of the the python matplotlib package." +matplotlib_repository_long_description = "This repository is intended to be defined as a complex repository dependency within a separate repository." + +numpy_repository_name = 'package_numpy_1_7_0170' +numpy_repository_description = "Contains a tool dependency definition that downloads and compiles version 1.7 of the the python numpy package." +numpy_repository_long_description = "This repository is intended to be defined as a complex repository dependency within a separate repository." + +category_name = 'Test 0170 Prior Installation Complex Dependencies' +category_description = 'Test 0170 Prior Installation Complex Dependencies' + +''' +1. Create and populate repositories package_matplotlib_1_2_0170 and package_numpy_1_7_0170. +2. Create a complex repository dependency on package_numpy_1_7_0170, and upload this to package_matplotlib_1_2_0170. +3. Verify that package_matplotlib_1_2_0170 now depends on package_numpy_1_7_0170, and that the inherited tool dependency displays correctly. +''' + + +class TestComplexPriorInstallation( ShedTwillTestCase ): + '''Test features related to datatype converters.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts.""" + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + + def test_0005_create_matplotlib_repository( self ): + '''Create and populate the package_matplotlib_1_2_0170 repository.''' + ''' + This is step 1 - Create and populate repositories package_matplotlib_1_2_0170 and package_numpy_1_7_0170. + ''' + category = self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + repository = self.get_or_create_repository( name=matplotlib_repository_name, + description=matplotlib_repository_description, + long_description=matplotlib_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='package_matplotlib/package_matplotlib_1_2.tar', + filepath=None, + valid_tools_only=False, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded matplotlib tool dependency tarball.', + strings_displayed=['orphan'], + strings_not_displayed=[] ) + + def test_0010_create_numpy_repository( self ): + '''Create and populate the package_numpy_1_7_0170 repository.''' + ''' + This is step 1 - Create and populate repositories package_matplotlib_1_2_0170 and package_numpy_1_7_0170. + ''' + category = self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + repository = self.get_or_create_repository( name=numpy_repository_name, + description=numpy_repository_description, + long_description=numpy_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='package_numpy/package_numpy_1_7.tar', + filepath=None, + valid_tools_only=False, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded numpy tool dependency tarball.', + strings_displayed=['orphan'], + strings_not_displayed=[] ) + + def test_0015_create_complex_repository_dependency( self ): + '''Create a dependency on package_numpy_1_7_0170.''' + ''' + This is step 2 - Create a complex repository dependency on package_numpy_1_7_0170, and upload this to package_matplotlib_1_2_0170. + package_matplotlib_1_2_0170 should depend on package_numpy_1_7_0170, with prior_installation_required + set to True. When matplotlib is selected for installation, the result should be that numpy is compiled + and installed first. + ''' + numpy_repository = test_db_util.get_repository_by_name_and_owner( numpy_repository_name, common.test_user_1_name ) + matplotlib_repository = test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) + # Generate the new dependency XML. Normally, the create_repository_dependency method would be used for this, but + # it replaces any existing tool or repository dependency XML file with the generated contents. This is undesirable + # in this case, because matplotlib already has an additional tool dependency definition that we don't want to + # overwrite. + new_xml = ' <package name="numpy" version="1.7">\n' + new_xml += ' <repository toolshed="%s" name="%s" owner="%s" changeset_revision="%s" prior_installation_required="True" />\n' + new_xml += ' </package>\n' + url = self.url + name = numpy_repository.name + owner = numpy_repository.user.username + changeset_revision = self.get_repository_tip( numpy_repository ) + processed_xml = new_xml % ( url, name, owner, changeset_revision ) + original_xml = file( self.get_filename( 'package_matplotlib/tool_dependencies.xml' ), 'r' ).read() + dependency_xml_path = self.generate_temp_path( 'test_0170', additional_paths=[ 'matplotlib' ] ) + new_xml_file = os.path.join( dependency_xml_path, 'tool_dependencies.xml' ) + file( new_xml_file, 'w' ).write( original_xml.replace( '', processed_xml ) ) + # Upload the generated complex repository dependency XML to the matplotlib repository. + self.upload_file( matplotlib_repository, + filename='tool_dependencies.xml', + filepath=dependency_xml_path, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded complex repository dependency on numpy 1.7.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0020_verify_generated_dependency( self ): + '''Verify that matplotlib now has a package tool dependency and a complex repository dependency.''' + ''' + This is step 3 - Verify that package_matplotlib_1_2_0170 now depends on package_numpy_1_7_0170, and that the inherited tool + dependency displays correctly. + 'Inhherited' in this case means that matplotlib should show a package tool dependency on numpy version 1.7, and a repository + dependency on the latest revision of package_numpy_1_7_0170. + ''' + numpy_repository = test_db_util.get_repository_by_name_and_owner( numpy_repository_name, common.test_user_1_name ) + matplotlib_repository = test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) + changeset_revision = self.get_repository_tip( numpy_repository ) + self.check_repository_dependency( matplotlib_repository, depends_on_repository=numpy_repository ) + self.display_manage_repository_page( matplotlib_repository, strings_displayed=[ 'numpy', '1.7', 'package', changeset_revision ] ) + + \ No newline at end of file diff -r a8139a367ed3de938f5d71336a06f1f768132469 -r 1037ab5b4f761c104b01cf0aee9222d28a67034b test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py --- a/test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py +++ b/test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py @@ -3,11 +3,11 @@ bwa_base_repository_name = 'bwa_base_repository_0100' bwa_base_repository_description = "BWA Base" -bwa_base_repository_long_description = "BWA tool that depends on bwa 0.5.9, with a complex repository dependency pointing at bwa_tool_repository_0100" +bwa_base_repository_long_description = "BWA tool that depends on bwa 0.5.9, with a complex repository dependency pointing at package_bwa_0_5_9_0100" -bwa_tool_repository_name = 'bwa_tool_repository_0100' -bwa_tool_repository_description = "BWA Tool" -bwa_tool_repository_long_description = "BWA repository with a package tool dependency defined for BWA 0.5.9." +bwa_package_repository_name = 'package_bwa_0_5_9_0100' +bwa_package_repository_description = "BWA Tool" +bwa_package_repository_long_description = "BWA repository with a package tool dependency defined to compile and install BWA 0.5.9." category_name = 'Test 0100 Complex Repository Dependencies' category_description = 'Test 0100 Complex Repository Dependencies' @@ -30,15 +30,15 @@ assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) - def test_0005_create_bwa_tool_repository( self ): + def test_0005_create_bwa_package_repository( self ): '''Create and populate bwa_tool_0100.''' global running_standalone category = self.create_category( name=category_name, description=category_description ) self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - repository = self.get_or_create_repository( name=bwa_tool_repository_name, - description=bwa_tool_repository_description, - long_description=bwa_tool_repository_long_description, + repository = self.get_or_create_repository( name=bwa_package_repository_name, + description=bwa_package_repository_description, + long_description=bwa_package_repository_long_description, owner=common.test_user_1_name, category_id=self.security.encode_id( category.id ), strings_displayed=[] ) @@ -73,7 +73,7 @@ owner=common.test_user_1_name, category_id=self.security.encode_id( category.id ), strings_displayed=[] ) - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) self.upload_file( repository, filename='bwa/complex/bwa_base.tar', filepath=None, @@ -91,7 +91,7 @@ dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'shed' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = 'http://http://this is not an url!' name = tool_repository.name owner = tool_repository.user.username @@ -113,7 +113,7 @@ dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'shed' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url name = 'invalid_repository!?' owner = tool_repository.user.username @@ -135,7 +135,7 @@ dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'shed' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url name = tool_repository.name owner = 'invalid_owner!?' @@ -157,7 +157,7 @@ dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'shed' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url name = tool_repository.name owner = tool_repository.user.username @@ -173,11 +173,11 @@ version='0.5.9' ) def test_0035_generate_valid_complex_repository_dependency( self ): - '''Generate and upload a valid tool_dependencies.xml file that specifies bwa_tool_repository_0100.''' + '''Generate and upload a valid tool_dependencies.xml file that specifies package_bwa_0_5_9_0100.''' global running_standalone if running_standalone: base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) url = self.url @@ -199,7 +199,7 @@ global running_standalone if running_standalone: base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) previous_changeset = self.get_repository_tip( tool_repository ) old_tool_dependency = self.get_filename( os.path.join( 'bwa', 'complex', 'readme', 'tool_dependencies.xml' ) ) new_tool_dependency_path = self.generate_temp_path( 'test_1100', additional_paths=[ 'tool_dependency' ] ) @@ -225,7 +225,7 @@ self.galaxy_logout() self.galaxy_login( email=common.admin_email, username=common.admin_username ) base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) preview_strings_displayed = [ tool_repository.name, self.get_repository_tip( tool_repository ) ] self.install_repository( bwa_base_repository_name, common.test_user_1_name, @@ -238,19 +238,19 @@ def test_0050_verify_installed_repositories( self ): '''Verify that the installed repositories are displayed properly.''' base_repository = test_db_util.get_installed_repository_by_name_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_installed_repository_by_name_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_installed_repository_by_name_owner( bwa_package_repository_name, common.test_user_1_name ) strings_displayed = [ 'bwa_base_repository_0100', 'user1', base_repository.installed_changeset_revision ] - strings_displayed.extend( [ 'bwa_tool_repository_0100', 'user1', tool_repository.installed_changeset_revision ] ) + strings_displayed.extend( [ 'package_bwa_0_5_9_0100', 'user1', tool_repository.installed_changeset_revision ] ) strings_displayed.append( self.url.replace( 'http://', '' ) ) self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=[] ) - strings_displayed = [ 'bwa_tool_repository_0100', 'user1', tool_repository.installed_changeset_revision ] + strings_displayed = [ 'package_bwa_0_5_9_0100', 'user1', tool_repository.installed_changeset_revision ] strings_not_displayed = [ 'Missing tool dependencies' ] self.display_installed_repository_manage_page( tool_repository, strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) strings_displayed = [ 'bwa_base_repository_0100', 'user1', - 'bwa_tool_repository_0100', + 'package_bwa_0_5_9_0100', base_repository.installed_changeset_revision, tool_repository.installed_changeset_revision ] strings_not_displayed = [ 'Missing tool dependencies' ] @@ -261,7 +261,7 @@ def test_0055_verify_complex_tool_dependency( self ): '''Verify that the generated env.sh contains the right data.''' base_repository = test_db_util.get_installed_repository_by_name_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_installed_repository_by_name_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_installed_repository_by_name_owner( bwa_package_repository_name, common.test_user_1_name ) env_sh_path = os.path.join( self.galaxy_tool_dependency_dir, 'bwa', '0.5.9', @@ -274,17 +274,17 @@ if tool_repository.installed_changeset_revision not in contents: raise AssertionError( 'Installed changeset revision %s not found in env.sh.\nContents of env.sh: %s' % \ ( tool_repository.installed_changeset_revision, contents ) ) - if 'bwa_tool_repository_0100' not in contents: - raise AssertionError( 'Repository name bwa_tool_repository_0100 not found in env.sh.\nContents of env.sh: %s' % contents ) + if 'package_bwa_0_5_9_0100' not in contents: + raise AssertionError( 'Repository name package_bwa_0_5_9_0100 not found in env.sh.\nContents of env.sh: %s' % contents ) def test_0060_verify_tool_dependency_uninstallation( self ): - '''Uninstall the bwa_tool_repository_0100 repository.''' + '''Uninstall the package_bwa_0_5_9_0100 repository.''' ''' Uninstall the repository that defines an orphan tool dependency on BWA 0.5.9, and verify that this results in the compiled binary package also being removed. ''' base_repository = test_db_util.get_installed_repository_by_name_owner( bwa_base_repository_name, common.test_user_1_name ) - tool_repository = test_db_util.get_installed_repository_by_name_owner( bwa_tool_repository_name, common.test_user_1_name ) + tool_repository = test_db_util.get_installed_repository_by_name_owner( bwa_package_repository_name, common.test_user_1_name ) self.uninstall_repository( tool_repository, remove_from_disk=True ) env_sh_path = os.path.join( self.galaxy_tool_dependency_dir, 'bwa', diff -r a8139a367ed3de938f5d71336a06f1f768132469 -r 1037ab5b4f761c104b01cf0aee9222d28a67034b test/tool_shed/functional/test_1170_complex_prior_installation_required.py --- /dev/null +++ b/test/tool_shed/functional/test_1170_complex_prior_installation_required.py @@ -0,0 +1,181 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +matplotlib_repository_name = 'package_matplotlib_1_2_0170' +matplotlib_repository_description = "Contains a tool dependency definition that downloads and compiles version 1.2.x of the the python matplotlib package." +matplotlib_repository_long_description = "This repository is intended to be defined as a complex repository dependency within a separate repository." + +numpy_repository_name = 'package_numpy_1_7_0170' +numpy_repository_description = "Contains a tool dependency definition that downloads and compiles version 1.7 of the the python numpy package." +numpy_repository_long_description = "This repository is intended to be defined as a complex repository dependency within a separate repository." + +category_name = 'Test 0170 Prior Installation Complex Dependencies' +category_description = 'Test 0170 Prior Installation Complex Dependencies' + +''' +1. Create and populate repositories package_matplotlib_1_2_0170 and package_numpy_1_7_0170. +2. Create a complex repository dependency on package_numpy_1_7_0170, and upload this to package_matplotlib_1_2_0170. +3. Verify that package_matplotlib_1_2_0170 now depends on package_numpy_1_7_0170, and that the inherited tool dependency displays correctly. +4. Install package_matplotlib_1_2_0170 with repository dependencies. +5. Verify that the prior_installation_required attribute resulted in package_numpy_1_7_0170 being installed first. +''' + +running_standalone = False + + +class TestComplexPriorInstallation( ShedTwillTestCase ): + '''Test features related to datatype converters.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts.""" + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + + def test_0005_create_matplotlib_repository( self ): + '''Create and populate the package_matplotlib_1_2_0170 repository.''' + ''' + This is step 1 - Create and populate repositories package_matplotlib_1_2_0170 and package_numpy_1_7_0170. + ''' + global running_standalone + category = self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + repository = self.get_or_create_repository( name=matplotlib_repository_name, + description=matplotlib_repository_description, + long_description=matplotlib_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + if self.repository_is_new( repository ): + running_standalone = True + self.upload_file( repository, + filename='package_matplotlib/package_matplotlib_1_2.tar', + filepath=None, + valid_tools_only=False, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded matplotlib tool dependency tarball.', + strings_displayed=['orphan'], + strings_not_displayed=[] ) + + def test_0010_create_numpy_repository( self ): + '''Create and populate the package_numpy_1_7_0170 repository.''' + ''' + This is step 1 - Create and populate repositories package_matplotlib_1_2_0170 and package_numpy_1_7_0170. + ''' + global running_standalone + category = self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + if running_standalone: + repository = self.get_or_create_repository( name=numpy_repository_name, + description=numpy_repository_description, + long_description=numpy_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='package_numpy/package_numpy_1_7.tar', + filepath=None, + valid_tools_only=False, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded numpy tool dependency tarball.', + strings_displayed=['orphan'], + strings_not_displayed=[] ) + + def test_0015_create_complex_repository_dependency( self ): + '''Create a dependency on package_numpy_1_7_0170.''' + ''' + This is step 2 - Create a complex repository dependency on package_numpy_1_7_0170, and upload this to package_matplotlib_1_2_0170. + package_matplotlib_1_2_0170 should depend on package_numpy_1_7_0170, with prior_installation_required + set to True. When matplotlib is selected for installation, the result should be that numpy is compiled + and installed first. + ''' + global running_standalone + numpy_repository = test_db_util.get_repository_by_name_and_owner( numpy_repository_name, common.test_user_1_name ) + matplotlib_repository = test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) + # Generate the new dependency XML. Normally, the create_repository_dependency method would be used for this, but + # it replaces any existing tool or repository dependency XML file with the generated contents. This is undesirable + # in this case, because matplotlib already has an additional tool dependency definition that we don't want to + # overwrite. + new_xml = ' <package name="numpy" version="1.7">\n' + new_xml += ' <repository toolshed="%s" name="%s" owner="%s" changeset_revision="%s" prior_installation_required="True" />\n' + new_xml += ' </package>\n' + url = self.url + name = numpy_repository.name + owner = numpy_repository.user.username + if running_standalone: + changeset_revision = self.get_repository_tip( numpy_repository ) + processed_xml = new_xml % ( url, name, owner, changeset_revision ) + original_xml = file( self.get_filename( 'package_matplotlib/tool_dependencies.xml' ), 'r' ).read() + dependency_xml_path = self.generate_temp_path( 'test_0170', additional_paths=[ 'matplotlib' ] ) + new_xml_file = os.path.join( dependency_xml_path, 'tool_dependencies.xml' ) + file( new_xml_file, 'w' ).write( original_xml.replace( '', processed_xml ) ) + # Upload the generated complex repository dependency XML to the matplotlib repository. + self.upload_file( matplotlib_repository, + filename='tool_dependencies.xml', + filepath=dependency_xml_path, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded complex repository dependency on numpy 1.7.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0020_verify_generated_dependency( self ): + '''Verify that matplotlib now has a package tool dependency and a complex repository dependency.''' + ''' + This is step 3 - Verify that package_matplotlib_1_2_0170 now depends on package_numpy_1_7_0170, and that the inherited tool + dependency displays correctly. + 'Inhherited' in this case means that matplotlib should show a package tool dependency on numpy version 1.7, and a repository + dependency on the latest revision of package_numpy_1_7_0170. + ''' + numpy_repository = test_db_util.get_repository_by_name_and_owner( numpy_repository_name, common.test_user_1_name ) + matplotlib_repository = test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) + changeset_revision = self.get_repository_tip( numpy_repository ) + self.check_repository_dependency( matplotlib_repository, depends_on_repository=numpy_repository ) + self.display_manage_repository_page( matplotlib_repository, strings_displayed=[ 'numpy', '1.7', 'package', changeset_revision ] ) + + def test_0025_install_matplotlib_repository( self ): + '''Install the package_matplotlib_1_2_0170 repository.''' + ''' + This is step 4 - Install package_matplotlib_1_2_0170 with repository dependencies. + ''' + self.galaxy_logout() + self.galaxy_login( email=common.admin_email, username=common.admin_username ) + matplotlib_repository = test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) + preview_strings_displayed = [ 'package_matplotlib_1_2_0170', self.get_repository_tip( matplotlib_repository ) ] + strings_displayed = [ 'Choose the tool panel section' ] + self.install_repository( matplotlib_repository_name, + common.test_user_1_name, + category_name, + install_tool_dependencies=False, + install_repository_dependencies=True, + preview_strings_displayed=preview_strings_displayed, + strings_displayed=[], + strings_not_displayed=[], + post_submit_strings_displayed=[ 'package_matplotlib_1_2_0170', 'new' ], + includes_tools_for_display_in_tool_panel=False ) + + def test_0030_verify_installation_order( self ): + '''Verify that the numpy repository was installed before the matplotlib repository.''' + ''' + This is step 5 - Verify that the prior_installation_required attribute resulted in package_numpy_1_7_0170 being installed first. + In the previous step, package_matplotlib_1_2_0170 was selected for installation, but package_numpy_1_7_0170 had the + prior_installation_required attribute set. Confirm that this resulted in package_numpy_1_7_0170 being installed before + package_matplotlib_1_2_0170. + ''' + matplotlib_repository = test_db_util.get_installed_repository_by_name_owner( matplotlib_repository_name, common.test_user_1_name ) + numpy_repository = test_db_util.get_installed_repository_by_name_owner( numpy_repository_name, common.test_user_1_name ) + assert matplotlib_repository.update_time > numpy_repository.update_time, \ + 'Error: package_numpy_1_7_0170 shows a later update time than package_matplotlib_1_2_0170' + \ No newline at end of file diff -r a8139a367ed3de938f5d71336a06f1f768132469 -r 1037ab5b4f761c104b01cf0aee9222d28a67034b test/tool_shed/test_data/package_matplotlib/package_matplotlib_1_2.tar Binary file test/tool_shed/test_data/package_matplotlib/package_matplotlib_1_2.tar has changed diff -r a8139a367ed3de938f5d71336a06f1f768132469 -r 1037ab5b4f761c104b01cf0aee9222d28a67034b test/tool_shed/test_data/package_matplotlib/tool_dependencies.xml --- /dev/null +++ b/test/tool_shed/test_data/package_matplotlib/tool_dependencies.xml @@ -0,0 +1,25 @@ +<tool_dependency> + + <package name="matplotlib" version="1.2.1"> + <install version="1.0"> + <actions> + <action type="download_by_url">https://downloads.sourceforge.net/project/matplotlib/matplotlib/matplotlib-…</action> + <action type="shell_command">wget http://downloads.sourceforge.net/project/freetype/freetype2/2.4.11/freetype…</action> + <action type="shell_command">tar xfvj freetype-2.4.11.tar.bz2 && + cd freetype-2.4.11 && + ./configure --prefix=$INSTALL_DIR/freetype/build && + make && + make install</action> + <action type="make_directory">$INSTALL_DIR/lib/python</action> + <action type="shell_command">export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR/lib/python && + export CPLUS_INCLUDE_PATH=$INSTALL_DIR/freetype/build/include:$INSTALL_DIR/freetype/build/include/freetype2/ && + export LIBRARY_PATH=$INSTALL_DIR/freetype/build/lib/ && + python setup.py install --home $INSTALL_DIR --install-scripts $INSTALL_DIR/bin</action> + <action type="set_environment"> + <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable> + </action> + </actions> + </install> + <readme>Compiling matplotlib requires a C compiler (typically gcc), freetype2, numpy and libpng.</readme> + </package> +</tool_dependency> diff -r a8139a367ed3de938f5d71336a06f1f768132469 -r 1037ab5b4f761c104b01cf0aee9222d28a67034b test/tool_shed/test_data/package_numpy/package_numpy_1_7.tar Binary file test/tool_shed/test_data/package_numpy/package_numpy_1_7.tar has changed Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Abbreviate not applicable in tool shed grids.
by commits-noreply＠bitbucket.org 22 Apr '13

22 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a8139a367ed3/ Changeset: a8139a367ed3 User: greg Date: 2013-04-22 16:52:29 Summary: Abbreviate not applicable in tool shed grids. Affected #: 1 file diff -r 77f32521f235ace1773e54389d7b536a9f5235a5 -r a8139a367ed3de938f5d71336a06f1f768132469 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -154,9 +154,9 @@ return 'yes' else: return 'no' - return 'not applicable' + return 'n/a' except: - return 'not applicable' + return 'n/a' class DescriptionColumn( grids.TextColumn ): @@ -748,9 +748,9 @@ return 'yes' else: return 'no' - return 'not applicable' + return 'n/a' except: - return 'not applicable' + return 'n/a' class DoNotTestColumn( grids.BooleanColumn ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Handle repositories whith no metadata revisions when displaying the verified tool column in the tool shed.
by commits-noreply＠bitbucket.org 22 Apr '13

22 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/77f32521f235/ Changeset: 77f32521f235 User: greg Date: 2013-04-22 16:50:10 Summary: Handle repositories whith no metadata revisions when displaying the verified tool column in the tool shed. Affected #: 1 file diff -r 5138b63cc96435ea72415796aa3daabfbfe2f23f -r 77f32521f235ace1773e54389d7b536a9f5235a5 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -147,13 +147,16 @@ def get_value( self, trans, grid, repository ): # This column will display the value associated with the currently displayed metadata revision. - displayed_metadata_revision = repository.metadata_revisions[ -1 ] - if displayed_metadata_revision.includes_tools: - if displayed_metadata_revision.tools_functionally_correct: - return 'yes' - else: - return 'no' - return 'not applicable' + try: + displayed_metadata_revision = repository.metadata_revisions[ -1 ] + if displayed_metadata_revision.includes_tools: + if displayed_metadata_revision.tools_functionally_correct: + return 'yes' + else: + return 'no' + return 'not applicable' + except: + return 'not applicable' class DescriptionColumn( grids.TextColumn ): @@ -735,13 +738,19 @@ class ToolsFunctionallyCorrectColumn( grids.BooleanColumn ): - def get_value( self, trans, grid, repository_metadata ): - if repository_metadata.includes_tools: - if repository_metadata.tools_functionally_correct: - return 'yes' - else: - return 'no' - return 'not applicable' + + def get_value( self, trans, grid, repository ): + # This column will display the value associated with the currently displayed metadata revision. + try: + displayed_metadata_revision = repository.metadata_revisions[ -1 ] + if displayed_metadata_revision.includes_tools: + if displayed_metadata_revision.tools_functionally_correct: + return 'yes' + else: + return 'no' + return 'not applicable' + except: + return 'not applicable' class DoNotTestColumn( grids.BooleanColumn ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Clean up the main repositories by category grid and add a tools verified column.
by commits-noreply＠bitbucket.org 22 Apr '13

22 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5138b63cc964/ Changeset: 5138b63cc964 User: greg Date: 2013-04-22 16:44:20 Summary: Clean up the main repositories by category grid and add a tools verified column. Affected #: 2 files diff -r cac1c2a9247bf1f2c7f78275a6d11bf209867fb2 -r 5138b63cc96435ea72415796aa3daabfbfe2f23f lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -61,6 +61,7 @@ my_writable_repositories_grid = repository_grids.MyWritableRepositoriesGrid() repositories_by_user_grid = repository_grids.RepositoriesByUserGrid() repositories_i_own_grid = repository_grids.RepositoriesIOwnGrid() + repositories_in_category_grid = repository_grids.RepositoriesInCategoryGrid() repository_dependencies_grid = repository_grids.RepositoryDependenciesGrid() repository_grid = repository_grids.RepositoryGrid() # The repository_metadata_grid is not currently displayed, but is sub-classed by several grids. @@ -202,13 +203,14 @@ elif operation == "my_writable_repositories": return self.my_writable_repositories_grid( trans, **kwd ) elif operation == "repositories_by_category": - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] category_id = kwd.get( 'id', None ) - category = suc.get_category( trans, category_id ) - kwd[ 'f-Category.name' ] = category.name + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_in_category', + id=category_id, + message=message, + status=status ) ) elif operation == "receive email alerts": if trans.user: if kwd[ 'id' ]: @@ -244,7 +246,7 @@ **kwd ) ) user_id = kwd.get( 'user_id', None ) if user_id is None: - # The received id is the repository id, so we need to get the id of the user that uploaded the repository. + # The received id is the repository id, so we need to get the id of the user that owns the repository. repository_id = kwd.get( 'id', None ) if repository_id: repository = suc.get_repository_in_tool_shed( trans, repository_id ) @@ -263,6 +265,40 @@ return self.repositories_by_user_grid( trans, **kwd ) @web.expose + def browse_repositories_in_category( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + if operation == 'repositories_by_user': + user_id = kwd.get( 'user_id', None ) + if user_id is None: + # The received id is the repository id, so we need to get the id of the user that owns the repository. + repository_id = kwd.get( 'id', None ) + if repository_id: + repository = suc.get_repository_in_tool_shed( trans, repository_id ) + user_id = trans.security.encode_id( repository.user.id ) + user = suc.get_user( trans, user_id ) + self.repositories_by_user_grid.title = "Repositories owned by %s" % user.username + kwd[ 'user_id' ] = user_id + return self.repositories_by_user_grid( trans, **kwd ) + selected_changeset_revision, repository = self.__get_repository_from_refresh_on_change( trans, **kwd ) + if repository: + # The user selected a repository revision which results in a refresh_on_change. + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + id=trans.security.encode_id( repository.id ), + changeset_revision=selected_changeset_revision ) ) + category_id = kwd.get( 'id', None ) + if category_id: + category = suc.get_category( trans, category_id ) + if category: + self.repositories_in_category_grid.title = 'Category %s' % str( category.name ) + return self.repositories_in_category_grid( trans, **kwd ) + + @web.expose def browse_repository( self, trans, id, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) diff -r cac1c2a9247bf1f2c7f78275a6d11bf209867fb2 -r 5138b63cc96435ea72415796aa3daabfbfe2f23f lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -143,6 +143,19 @@ return escape_html( repository.revision( trans.app ) ) + class ToolsFunctionallyCorrectColumn( grids.BooleanColumn ): + + def get_value( self, trans, grid, repository ): + # This column will display the value associated with the currently displayed metadata revision. + displayed_metadata_revision = repository.metadata_revisions[ -1 ] + if displayed_metadata_revision.includes_tools: + if displayed_metadata_revision.tools_functionally_correct: + return 'yes' + else: + return 'no' + return 'not applicable' + + class DescriptionColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): @@ -261,6 +274,60 @@ .outerjoin( model.Category.table ) +class RepositoriesInCategoryGrid( RepositoryGrid ): + title = "Category" + + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( controller="repository", operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryGrid.DescriptionColumn( "Synopsis", + key="description", + attach_popup=False ), + RepositoryGrid.MetadataRevisionColumn( "Metadata Revisions" ), + RepositoryGrid.ToolsFunctionallyCorrectColumn( "Tools Verified" ), + RepositoryGrid.UserColumn( "Owner", + model_class=model.User, + link=( lambda item: dict( controller="repository", operation="repositories_by_user", id=item.id ) ), + attach_popup=False, + key="User.username" ), + # Columns that are valid for filtering but are not visible. + RepositoryGrid.EmailColumn( "Email", + model_class=model.User, + key="email", + visible=False ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name, description", + cols_to_filter=[ columns[0], columns[1] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ grids.GridOperation( "Receive email alerts", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) ] + + def build_initial_query( self, trans, **kwd ): + category_id = kwd.get( 'id', None ) + if category_id: + category = suc.get_category( trans, category_id ) + if category: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) \ + .filter( model.Category.table.c.name == category.name ) + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + + class RepositoriesByUserGrid( RepositoryGrid ): title = "Repositories by user" columns = [ @@ -268,11 +335,11 @@ key="name", link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), attach_popup=False ), - RepositoryGrid.MetadataRevisionColumn( "Metadata Revisions" ), - RepositoryGrid.TipRevisionColumn( "Tip Revision" ), RepositoryGrid.DescriptionColumn( "Synopsis", key="description", attach_popup=False ), + RepositoryGrid.MetadataRevisionColumn( "Metadata Revisions" ), + RepositoryGrid.ToolsFunctionallyCorrectColumn( "Tools Verified" ), RepositoryGrid.CategoryColumn( "Category", model_class=model.Category, key="Category.name", @@ -304,7 +371,7 @@ link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), attach_popup=True ), RepositoryGrid.MetadataRevisionColumn( "Metadata Revisions" ), - RepositoryGrid.TipRevisionColumn( "Tip Revision" ), + RepositoryGrid.ToolsFunctionallyCorrectColumn( "Tools Verified" ), RepositoryGrid.CategoryColumn( "Category", model_class=model.Category, key="Category.name", @@ -343,7 +410,7 @@ link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), attach_popup=True ), RepositoriesIOwnGrid.MetadataRevisionColumn( "Metadata Revisions" ), - RepositoriesIOwnGrid.TipRevisionColumn( "Tip Revision" ), + RepositoryGrid.ToolsFunctionallyCorrectColumn( "Tools Verified" ), RepositoriesIOwnGrid.CategoryColumn( "Category", model_class=model.Category, key="Category.name", @@ -403,7 +470,7 @@ link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), attach_popup=True ), RepositoryGrid.MetadataRevisionColumn( "Metadata Revisions" ), - RepositoryGrid.TipRevisionColumn( "Tip Revision" ), + RepositoryGrid.ToolsFunctionallyCorrectColumn( "Tools Verified" ), RepositoryGrid.UserColumn( "Owner", model_class=model.User, link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), @@ -480,7 +547,7 @@ return query.filter( model.Category.name == column_filter ) - class RevisionColumn( grids.GridColumn ): + class InstallableRevisionColumn( grids.GridColumn ): def __init__( self, col_name ): grids.GridColumn.__init__( self, col_name ) @@ -502,7 +569,8 @@ RepositoryGrid.DescriptionColumn( "Synopsis", key="description", attach_popup=False ), - RevisionColumn( "Installable Revisions" ), + InstallableRevisionColumn( "Installable Revisions" ), + RepositoryGrid.ToolsFunctionallyCorrectColumn( "Tools Verified" ), RepositoryGrid.UserColumn( "Owner", model_class=model.User, attach_popup=False ), @@ -668,9 +736,12 @@ class ToolsFunctionallyCorrectColumn( grids.BooleanColumn ): def get_value( self, trans, grid, repository_metadata ): - if repository_metadata.tools_functionally_correct: - return 'yes' - return '' + if repository_metadata.includes_tools: + if repository_metadata.tools_functionally_correct: + return 'yes' + else: + return 'no' + return 'not applicable' class DoNotTestColumn( grids.BooleanColumn ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 20 Apr '13

20 Apr '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/cac1c2a9247b/ Changeset: cac1c2a9247b User: jgoecks Date: 2013-04-20 21:04:07 Summary: (a) Fetch sample names before drawing variant track and (b) update dataset metadata when changed. Affected #: 2 files diff -r 9b46c246c9cffbf9cbaaf1398e3d1cf51a3c42f6 -r cac1c2a9247bf1f2c7f78275a6d11bf209867fb2 static/scripts/mvc/data.js --- a/static/scripts/mvc/data.js +++ b/static/scripts/mvc/data.js @@ -20,8 +20,15 @@ }, initialize: function() { - // -- Create and initialize metadata. -- + // Set metadata. + // FIXME: pass back a metadata dict and then Backbone-relational + // can be used unpack metadata automatically. + this._set_metadata(); + // Update metadata on change. + this.on('change', this._set_metadata, this); + }, + _set_metadata: function() { var metadata = new DatasetMetadata(); // Move metadata from dataset attributes to metadata object. @@ -34,7 +41,8 @@ } }, this); - this.set('metadata', metadata); + // Because this is an internal change, silence it. + this.set('metadata', metadata, { 'silent': true }); }, /** diff -r 9b46c246c9cffbf9cbaaf1398e3d1cf51a3c42f6 -r cac1c2a9247bf1f2c7f78275a6d11bf209867fb2 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -4263,6 +4263,15 @@ }, /** + * Additional initialization required before drawing track for the first time. + */ + predraw_init: function() { + if (!this.dataset.get_metadata('sample_names')) { + return this.dataset.fetch(); + } + }, + + /** * Actions to be taken after draw has been completed. Draw is completed when all tiles have been * drawn/fetched and shown. */ https://bitbucket.org/galaxy/galaxy-central/commits/9b46c246c9cf/ Changeset: 9b46c246c9cf User: jgoecks Date: 2013-04-20 20:28:11 Summary: Trackster: refactor variant sample labels and fix line-height issues. Affected #: 1 file diff -r 71fdd146b95ce53735003afa282f51fa188398ad -r 9b46c246c9cffbf9cbaaf1398e3d1cf51a3c42f6 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -4271,45 +4271,44 @@ // Add summary/sample labels if needed and not already included. if ( !(tiles[0] instanceof SummaryTreeTile) && this.prefs.show_labels) { + var font_size; + // Add and/or style labels. if (this.container_div.find('.yaxislabel.variant').length === 0) { // Add summary and sample labels. - // FIXME: label attributes could be cleaner by using CSS classes. - - // Add summary label. - var summary_div_font_size = 10, - summary_div = $("<div/>").text('Summary').addClass('yaxislabel variant top').css({ - 'font-size': summary_div_font_size + 'px' - }); - this.container_div.prepend(summary_div); - - // Adjust summary label to middle of summary. - var base_offset = summary_div.position().top; - summary_div.css('top', base_offset + (this.prefs.summary_height - summary_div_font_size) / 2 + 'px'); - + // Add summary label to middle of summary area. + font_size = this.prefs.summary_height / 2; + this.tiles_div.prepend( + $("<div/>").text('Summary').addClass('yaxislabel variant top').css({ + 'font-size': font_size + 'px', + 'top': (this.prefs.summary_height - font_size) / 2 + 'px' + }) + ); + // Show sample labels. if (this.prefs.show_sample_data) { - var samples_div_html = ''; - _.each(this.dataset.get('metadata').get('sample_names'), function(name) { - samples_div_html += (name + '<br>'); - }); + var samples_div_html = this.dataset.get('metadata').get('sample_names').join('<br/>'); - var samples_div = $("<div/>").html(samples_div_html).addClass('yaxislabel variant top sample').css({ - // +2 for padding - 'top': base_offset + this.prefs.summary_height + 2, - }); - this.container_div.prepend(samples_div); + this.tiles_div.prepend( + $("<div/>").html(samples_div_html).addClass('yaxislabel variant top sample').css({ + // +2 for padding + 'top': this.prefs.summary_height + 2, + }) + ); } } // Style labels. // Match sample font size to mode. - $(this.container_div).find('.sample').css('font-size', (this.mode === 'Squish' ? 5 : 10) + 'px'); + font_size = (this.mode === 'Squish' ? 5 : 10) + 'px'; + $(this.tiles_div).find('.sample').css({ + 'font-size': font_size, + 'line-height': font_size + }); // Color labels to preference color. - $(this.container_div).find('.yaxislabel').css('color', this.prefs.label_color); - + $(this.tiles_div).find('.yaxislabel').css('color', this.prefs.label_color); } else { // Remove all labels. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 20 Apr '13

20 Apr '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/71fdd146b95c/ Changeset: 71fdd146b95c User: jgoecks Date: 2013-04-20 17:05:42 Summary: Visual analysis enhancements: (a) do not manually convert to json because it is not necessary and (b) gracefully handle missing sample names. Affected #: 2 files diff -r 034b2d6118caa1dc6808f1570e92d927973a87ed -r 71fdd146b95ce53735003afa282f51fa188398ad lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -193,7 +193,7 @@ original_dataset = self.get_dataset( trans, payload[ 'target_dataset_id' ], check_ownership=False, check_accessible=True ) msg = self.check_dataset_state( trans, original_dataset ) if msg: - return to_json_string( msg ) + return msg # # Set tool parameters--except non-hidden dataset parameters--using combination of @@ -223,7 +223,8 @@ for jida in original_job.input_datasets: input_dataset = jida.dataset data_provider = data_provider_registry.get_data_provider( trans, original_dataset=input_dataset, source='data' ) - if data_provider and not data_provider.converted_dataset: + if data_provider and ( not data_provider.converted_dataset + or data_provider.converted_dataset.state != trans.app.model.Dataset.states.OK ): # Can convert but no converted dataset yet, so return message about why. data_sources = input_dataset.datatype.data_sources msg = input_dataset.convert_dataset( trans, data_sources[ 'data' ] ) @@ -233,7 +234,7 @@ # Return any messages generated during conversions. return_message = self._get_highest_priority_msg( messages_list ) if return_message: - return to_json_string( return_message ) + return return_message # # Set target history (the history that tool will use for inputs/outputs). @@ -371,7 +372,7 @@ # Add dataset to tool's parameters. if not set_param_value( tool_params, jida.name, subset_dataset ): - return to_json_string( { "error" : True, "message" : "error setting parameter %s" % jida.name } ) + return { "error" : True, "message" : "error setting parameter %s" % jida.name } # # Execute tool and handle outputs. @@ -382,7 +383,7 @@ job_params={ "source" : "trackster" } ) except Exception, e: # Lots of things can go wrong when trying to execute tool. - return to_json_string( { "error" : True, "message" : e.__class__.__name__ + ": " + str(e) } ) + return { "error" : True, "message" : e.__class__.__name__ + ": " + str(e) } if run_on_regions: for output in subset_job_outputs.values(): output.visible = False diff -r 034b2d6118caa1dc6808f1570e92d927973a87ed -r 71fdd146b95ce53735003afa282f51fa188398ad static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1821,7 +1821,7 @@ // Start with this status message. //new_track.container_div.addClass("pending"); - //new_track.content_div.text("Converting input data so that it can be used quickly with tool."); + //new_track.content_div.html(DATA_PENDING); $.when(ss_deferred.go()).then(function(response) { if (response === "no converter") { @@ -4234,7 +4234,23 @@ } else { var dummy_painter = new (this.painter)(null, null, null, this.prefs, mode); - return dummy_painter.get_required_height(this.dataset.get_metadata('sample_names').length); + // HACK: sample_names is not be defined when dataset definition is fetched before + // dataset is complete (as is done when running tools). In that case, fall back on + // # of samples in data. This can be fixed by re-requesting dataset definition + // in init. + var num_samples = ( this.dataset.get_metadata('sample_names') ? this.dataset.get_metadata('sample_names').length : 0); + if (num_samples === 0 && result.data.length !== 0) { + // Sample data is separated by commas, so this computes # of samples: + num_samples = result.data[0][7].match(/,/g); + if ( num_samples === null ) { + num_samples = 1; + } + else { + num_samples = num_samples.length + 1; + } + } + + return dummy_painter.get_required_height(num_samples); } }, https://bitbucket.org/galaxy/galaxy-central/commits/034b2d6118ca/ Changeset: 034b2d6118ca User: jgoecks Date: 2013-04-20 16:08:29 Summary: Remove debugging statement Affected #: 1 file diff -r 1fa1583641474f160ca1569ec91faf8f3d7093c8 -r 034b2d6118caa1dc6808f1570e92d927973a87ed lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -796,7 +796,6 @@ # Search for and yield other data lines. for data_line in source: if line_in_region( data_line, chrom, start, end ): - print chrom, start, end, ">>>", data_line, yield data_line return line_filter_iter() https://bitbucket.org/galaxy/galaxy-central/commits/1fa158364147/ Changeset: 1fa158364147 User: jgoecks Date: 2013-04-20 16:06:26 Summary: Use tabix rather than interval index for pileup data indexing. Affected #: 2 files diff -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 -r 1fa1583641474f160ca1569ec91faf8f3d7093c8 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -160,7 +160,8 @@ </datatype><datatype extension="pdf" type="galaxy.datatypes.images:Pdf" mimetype="application/pdf"/><datatype extension="pileup" type="galaxy.datatypes.tabular:Pileup" display_in_upload="true"> - <converter file="pileup_to_interval_index_converter.xml" target_datatype="interval_index"/> + <converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/> + <converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/></datatype><datatype extension="png" type="galaxy.datatypes.images:Png" mimetype="image/png"/><datatype extension="qual" type="galaxy.datatypes.qualityscore:QualityScore" /> diff -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 -r 1fa1583641474f160ca1569ec91faf8f3d7093c8 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -474,11 +474,12 @@ """Tab delimited data in pileup (6- or 10-column) format""" file_ext = "pileup" line_class = "genomic coordinate" - data_sources = { "data": "interval_index" } + data_sources = { "data": "tabix" } """Add metadata elements""" MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter ) MetadataElement( name="startCol", default=2, desc="Start column", param=metadata.ColumnParameter ) + MetadataElement( name="endCol", default=2, desc="End column", param=metadata.ColumnParameter ) MetadataElement( name="baseCol", default=3, desc="Reference base column", param=metadata.ColumnParameter ) def init_meta( self, dataset, copy_from=None ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Refactor repository dependency generation. Add functional tests for the recently introduced feature to determine installation order of dependencies.
by commits-noreply＠bitbucket.org 19 Apr '13

19 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8d72c9adccf9/ Changeset: 8d72c9adccf9 User: inithello Date: 2013-04-19 21:59:09 Summary: Refactor repository dependency generation. Add functional tests for the recently introduced feature to determine installation order of dependencies. Affected #: 31 files diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/base/common.py --- a/test/tool_shed/base/common.py +++ b/test/tool_shed/base/common.py @@ -32,7 +32,7 @@ </repositories> ''' -new_repository_dependencies_line = ''' <repository toolshed="${toolshed_url}" name="${repository_name}" owner="${owner}" changeset_revision="${changeset_revision}" />''' +new_repository_dependencies_line = ''' <repository toolshed="${toolshed_url}" name="${repository_name}" owner="${owner}" changeset_revision="${changeset_revision}"${prior_installation_required} />''' # Set a 3 minute timeout for repository installation. This should be sufficient, since we're not installing tool dependencies. repository_installation_timeout = 180 \ No newline at end of file diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -254,29 +254,37 @@ else: return '%s=%s' % ( field_name, field_value ) - def create_repository_complex_dependency( self, repository, xml_filename, depends_on={} ): - self.generate_repository_dependency_xml( depends_on[ 'repositories' ], - xml_filename, - complex=True, - package=depends_on[ 'package' ], - version=depends_on[ 'version' ] ) + def create_repository_dependency( self, + repository=None, + repository_tuples=[], + filepath=None, + prior_installation_required=False, + complex=False, + package=None, + version=None, + strings_displayed=[], + strings_not_displayed=[] ): + repository_names = [] + if complex: + filename = 'tool_dependencies.xml' + self.generate_complex_dependency_xml( filename=filename, filepath=filepath, repository_tuples=repository_tuples, package=package, version=version ) + else: + for toolshed_url, name, owner, changeset_revision in repository_tuples: + repository_names.append( name ) + dependency_description = '%s depends on %s.' % ( repository.name, ', '.join( repository_names ) ) + filename = 'repository_dependencies.xml' + self.generate_simple_dependency_xml( repository_tuples=repository_tuples, + filename=filename, + filepath=filepath, + dependency_description=dependency_description, + prior_installation_required=prior_installation_required ) self.upload_file( repository, - 'tool_dependencies.xml', - filepath=os.path.split( xml_filename )[0], - commit_message='Uploaded dependency on %s.' % ', '.join( repo.name for repo in depends_on[ 'repositories' ] ) ) - - def create_repository_dependency( self, repository=None, depends_on=[], filepath=None ): - dependency_description = '%s depends on %s.' % ( repository.name, ', '.join( repo.name for repo in depends_on ) ) - self.generate_repository_dependency_xml( depends_on, - self.get_filename( 'repository_dependencies.xml', filepath=filepath ), - dependency_description=dependency_description ) - self.upload_file( repository, - 'repository_dependencies.xml', + filename=filename, filepath=filepath, - valid_tools_only=True, + valid_tools_only=False, uncompress_file=False, remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on %s.' % ', '.join( repo.name for repo in depends_on ), + commit_message='Uploaded dependency on %s.' % ', '.join( repository_names ), strings_displayed=[], strings_not_displayed=[] ) @@ -512,53 +520,55 @@ self.check_page_for_string( "You have been logged out" ) self.home() - def generate_invalid_dependency_xml( self, xml_filename, url, name, owner, changeset_revision, complex=True, package=None, version=None, description=None ): - file_path = os.path.split( xml_filename )[0] + def generate_complex_dependency_xml( self, filename, filepath, repository_tuples, package, version ): + file_path = os.path.join( filepath, filename ) dependency_entries = [] template = string.Template( common.new_repository_dependencies_line ) - dependency_entries.append( template.safe_substitute( toolshed_url=url, - owner=owner, - repository_name=name, - changeset_revision=changeset_revision ) ) - if not os.path.exists( file_path ): - os.makedirs( file_path ) - if complex: - dependency_template = string.Template( common.complex_repository_dependency_template ) - repository_dependency_xml = dependency_template.safe_substitute( package=package, version=version, dependency_lines='\n'.join( dependency_entries ) ) + for toolshed_url, name, owner, changeset_revision in repository_tuples: + dependency_entries.append( template.safe_substitute( toolshed_url=toolshed_url, + owner=owner, + repository_name=name, + changeset_revision=changeset_revision, + prior_installation_required='' ) ) + if not os.path.exists( filepath ): + os.makedirs( filepath ) + dependency_template = string.Template( common.complex_repository_dependency_template ) + repository_dependency_xml = dependency_template.safe_substitute( package=package, version=version, dependency_lines='\n'.join( dependency_entries ) ) + # Save the generated xml to the specified location. + file( file_path, 'w' ).write( repository_dependency_xml ) + + def generate_simple_dependency_xml( self, + repository_tuples, + filename, + filepath, + dependency_description='', + complex=False, + package=None, + version=None, + prior_installation_required=False ): + if not os.path.exists( filepath ): + os.makedirs( filepath ) + dependency_entries = [] + if prior_installation_required: + prior_installation_value = ' prior_installation_required="True"' else: - if not description: - description = ' description=""' - else: - description = ' description="%s"' % description - template_parser = string.Template( common.new_repository_dependencies_xml ) - repository_dependency_xml = template_parser.safe_substitute( description=description, dependency_lines='\n'.join( dependency_entries ) ) - # Save the generated xml to the specified location. - file( xml_filename, 'w' ).write( repository_dependency_xml ) - - def generate_repository_dependency_xml( self, repositories, xml_filename, dependency_description='', complex=False, package=None, version=None ): - file_path = os.path.split( xml_filename )[0] - if not os.path.exists( file_path ): - os.makedirs( file_path ) - dependency_entries = [] - for repository in repositories: - changeset_revision = self.get_repository_tip( repository ) + prior_installation_value = '' + for toolshed_url, name, owner, changeset_revision in repository_tuples: template = string.Template( common.new_repository_dependencies_line ) - dependency_entries.append( template.safe_substitute( toolshed_url=self.url, - owner=repository.user.username, - repository_name=repository.name, - changeset_revision=changeset_revision ) ) + dependency_entries.append( template.safe_substitute( toolshed_url=toolshed_url, + owner=owner, + repository_name=name, + changeset_revision=changeset_revision, + prior_installation_required=prior_installation_value ) ) if dependency_description: description = ' description="%s"' % dependency_description else: description = dependency_description - if complex: - dependency_template = string.Template( common.complex_repository_dependency_template ) - repository_dependency_xml = dependency_template.safe_substitute( package=package, version=version, dependency_lines='\n'.join( dependency_entries ) ) - else: - template_parser = string.Template( common.new_repository_dependencies_xml ) - repository_dependency_xml = template_parser.safe_substitute( description=description, dependency_lines='\n'.join( dependency_entries ) ) + template_parser = string.Template( common.new_repository_dependencies_xml ) + repository_dependency_xml = template_parser.safe_substitute( description=description, dependency_lines='\n'.join( dependency_entries ) ) # Save the generated xml to the specified location. - file( xml_filename, 'w' ).write( repository_dependency_xml ) + full_path = os.path.join( filepath, filename ) + file( full_path, 'w' ).write( repository_dependency_xml ) def generate_temp_path( self, test_script_path, additional_paths=[] ): temp_path = os.path.join( self.tool_shed_test_tmp_dir, test_script_path, os.sep.join( additional_paths ) ) diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0020_basic_repository_dependencies.py --- a/test/tool_shed/functional/test_0020_basic_repository_dependencies.py +++ b/test/tool_shed/functional/test_0020_basic_repository_dependencies.py @@ -9,8 +9,10 @@ emboss_repository_description = 'Galaxy wrappers for Emboss version 5.0.0 tools for test 0020' emboss_repository_long_description = 'Galaxy wrappers for Emboss version 5.0.0 tools for test 0020' + class TestBasicRepositoryDependencies( ShedTwillTestCase ): '''Testing emboss 5 with repository dependencies.''' + def test_0000_initiate_users( self ): """Create necessary user accounts and login as an admin user.""" self.logout() @@ -23,9 +25,11 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_category( self ): """Create a category for this test suite""" self.create_category( name='Test 0020 Basic Repository Dependencies', description='Testing basic repository dependency features.' ) + def test_0010_create_emboss_datatypes_repository_and_upload_tarball( self ): '''Create and populate the emboss_datatypes repository.''' self.logout() @@ -46,10 +50,12 @@ commit_message='Uploaded datatypes_conf.xml.', strings_displayed=[], strings_not_displayed=[] ) + def test_0015_verify_datatypes_in_datatypes_repository( self ): '''Verify that the emboss_datatypes repository contains datatype entries.''' repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) self.display_manage_repository_page( repository, strings_displayed=[ 'Datatypes', 'equicktandem', 'hennig86', 'vectorstrip' ] ) + def test_0020_create_emboss_5_repository_and_upload_files( self ): '''Create and populate the emboss_5_0020 repository.''' category = test_db_util.get_category_by_name( 'Test 0020 Basic Repository Dependencies' ) @@ -68,22 +74,15 @@ commit_message='Uploaded emboss.tar', strings_displayed=[], strings_not_displayed=[] ) + def test_0025_generate_and_upload_repository_dependencies_xml( self ): '''Generate and upload the repository_dependencies.xml file''' repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0020', additional_paths=[ 'emboss', '5' ] ) - self.generate_repository_dependency_xml( [ datatypes_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ) ) - self.upload_file( repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded repository_dependencies.xml.', - strings_displayed=[], - strings_not_displayed=[] ) + repository_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) + self.create_repository_dependency( repository=repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) + def test_0030_verify_emboss_5_dependencies( self ): '''Verify that the emboss_5 repository now depends on the emboss_datatypes repository with correct name, owner, and changeset revision.''' repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) @@ -98,6 +97,7 @@ changeset_revision, 'Repository dependencies' ] self.display_manage_repository_page( repository, strings_displayed=strings_displayed ) + def test_0040_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0030_repository_dependency_revisions.py --- a/test/tool_shed/functional/test_0030_repository_dependency_revisions.py +++ b/test/tool_shed/functional/test_0030_repository_dependency_revisions.py @@ -1,7 +1,6 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os import tool_shed.base.test_db_util as test_db_util -import logging -log = logging.getLogger(__name__) + datatypes_repository_name = 'emboss_datatypes_0030' datatypes_repository_description = "Galaxy applicable data formats used by Emboss tools." datatypes_repository_long_description = "Galaxy applicable data formats used by Emboss tools. This repository contains no tools." @@ -12,8 +11,10 @@ emboss_repository_description = 'Galaxy wrappers for Emboss version 5.0.0 tools for test 0030' emboss_repository_long_description = 'Galaxy wrappers for Emboss version 5.0.0 tools for test 0030' + class TestRepositoryDependencyRevisions( ShedTwillTestCase ): '''Test dependencies on different revisions of a repository.''' + def test_0000_initiate_users( self ): """Create necessary user accounts.""" self.logout() @@ -26,9 +27,11 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_category( self ): """Create a category for this test suite""" self.create_category( name='Test 0030 Repository Dependency Revisions', description='Testing repository dependencies by revision.' ) + def test_0010_create_emboss_5_repository( self ): '''Create and populate the emboss_5_0030 repository.''' self.logout() @@ -48,6 +51,7 @@ commit_message='Uploaded tool tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0015_create_emboss_6_repository( self ): '''Create and populate the emboss_6_0030 repository.''' self.logout() @@ -67,6 +71,7 @@ commit_message='Uploaded tool tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0020_create_emboss_datatypes_repository( self ): '''Create and populate the emboss_datatypes_0030 repository.''' self.logout() @@ -86,6 +91,7 @@ commit_message='Uploaded datatypes_conf.xml.', strings_displayed=[], strings_not_displayed=[] ) + def test_0025_create_emboss_repository( self ): '''Create and populate the emboss_0030 repository.''' self.logout() @@ -105,69 +111,39 @@ commit_message='Uploaded the tool tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0030_generate_repository_dependencies_for_emboss_5( self ): '''Generate a repository_dependencies.xml file specifying emboss_datatypes and upload it to the emboss_5 repository.''' datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) - repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss' ] ) - self.generate_repository_dependency_xml( [ datatypes_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ) ) emboss_5_repository = test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) - self.upload_file( emboss_5_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded repository_dependencies.xml.', - strings_displayed=[], - strings_not_displayed=[] ) + repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss5' ] ) + datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) + self.create_repository_dependency( repository=emboss_5_repository, repository_tuples=[ datatypes_tuple ], filepath=repository_dependencies_path ) + def test_0035_generate_repository_dependencies_for_emboss_6( self ): '''Generate a repository_dependencies.xml file specifying emboss_datatypes and upload it to the emboss_6 repository.''' emboss_6_repository = test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) - repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss' ] ) - self.upload_file( emboss_6_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded repository_dependencies.xml.', - strings_displayed=[], - strings_not_displayed=[] ) + datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss6' ] ) + datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) + self.create_repository_dependency( repository=emboss_6_repository, repository_tuples=[ datatypes_tuple ], filepath=repository_dependencies_path ) + def test_0040_generate_repository_dependency_on_emboss_5( self ): '''Create and upload repository_dependencies.xml for the emboss_5_0030 repository.''' emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) emboss_5_repository = test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss', '5' ] ) - self.generate_repository_dependency_xml( [ emboss_5_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Emboss requires the Emboss 5 repository.' ) - self.upload_file( emboss_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded repository_dependencies.xml.', - strings_displayed=[], - strings_not_displayed=[] ) + emboss_tuple = ( self.url, emboss_5_repository.name, emboss_5_repository.user.username, self.get_repository_tip( emboss_5_repository ) ) + self.create_repository_dependency( repository=emboss_repository, repository_tuples=[ emboss_tuple ], filepath=repository_dependencies_path ) + def test_0045_generate_repository_dependency_on_emboss_6( self ): '''Create and upload repository_dependencies.xml for the emboss_6_0030 repository.''' emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) emboss_6_repository = test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) - repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss', '6' ] ) - self.generate_repository_dependency_xml( [ emboss_6_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Emboss requires the Emboss 6 repository.' ) - self.upload_file( emboss_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded repository_dependencies.xml.', - strings_displayed=[], - strings_not_displayed=[] ) + repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss', '5' ] ) + emboss_tuple = ( self.url, emboss_6_repository.name, emboss_6_repository.user.username, self.get_repository_tip( emboss_6_repository ) ) + self.create_repository_dependency( repository=emboss_repository, repository_tuples=[ emboss_tuple ], filepath=repository_dependencies_path ) + def test_0050_verify_repository_dependency_revisions( self ): '''Verify that different metadata revisions of the emboss repository have different repository dependencies.''' repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) @@ -178,7 +154,7 @@ # Iterate through all metadata revisions and check for repository dependencies. for metadata, changeset_revision in repository_metadata: # Add the dependency description and datatypes repository details to the strings to check. - strings_displayed = [ 'Emboss requires the Emboss', 'emboss_datatypes_0030', 'user1', datatypes_tip ] + strings_displayed = [ 'emboss_datatypes_0030', 'user1', datatypes_tip ] strings_displayed.extend( [ 'Tool dependencies', 'emboss', '5.0.0', 'package' ] ) self.display_manage_repository_page( repository, changeset_revision=changeset_revision, diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0040_repository_circular_dependencies.py --- a/test/tool_shed/functional/test_0040_repository_circular_dependencies.py +++ b/test/tool_shed/functional/test_0040_repository_circular_dependencies.py @@ -9,8 +9,10 @@ filtering_repository_description = "Galaxy's filtering tool for test 0040" filtering_repository_long_description = "Long description of Galaxy's filtering tool for test 0040" + class TestRepositoryCircularDependencies( ShedTwillTestCase ): '''Verify that the code correctly displays repositories with circular repository dependencies.''' + def test_0000_initiate_users( self ): """Create necessary user accounts.""" self.logout() @@ -23,9 +25,11 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_category( self ): """Create a category for this test suite""" self.create_category( name='test_0040_repository_circular_dependencies', description='Testing handling of circular repository dependencies.' ) + def test_0010_create_freebayes_repository( self ): '''Create and populate freebayes_0040.''' self.logout() @@ -45,6 +49,7 @@ commit_message='Uploaded the tool tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0015_create_filtering_repository( self ): '''Create and populate filtering_0040.''' self.logout() @@ -64,6 +69,7 @@ commit_message='Uploaded the tool tarball for filtering 1.1.0.', strings_displayed=[], strings_not_displayed=[] ) + def test_0020_create_dependency_on_freebayes( self ): '''Upload a repository_dependencies.xml file that specifies the current revision of freebayes to the filtering_0040 repository.''' # The dependency structure should look like: @@ -73,18 +79,9 @@ repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0040', additional_paths=[ 'filtering' ] ) - self.generate_repository_dependency_xml( [ repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Filtering 1.1.0 depends on the freebayes repository.' ) - self.upload_file( filtering_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on freebayes.', - strings_displayed=[], - strings_not_displayed=[] ) + repository_tuple = ( self.url, repository.name, repository.user.username, self.get_repository_tip( repository ) ) + self.create_repository_dependency( repository=filtering_repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) + def test_0025_create_dependency_on_filtering( self ): '''Upload a repository_dependencies.xml file that specifies the current revision of filtering to the freebayes_0040 repository.''' # The dependency structure should look like: @@ -94,18 +91,9 @@ repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0040', additional_paths=[ 'freebayes' ] ) - self.generate_repository_dependency_xml( [ repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Freebayes depends on the filtering repository.' ) - self.upload_file( freebayes_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on filtering.', - strings_displayed=[], - strings_not_displayed=[] ) + repository_tuple = ( self.url, repository.name, repository.user.username, self.get_repository_tip( repository ) ) + self.create_repository_dependency( repository=freebayes_repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) + def test_0030_verify_repository_dependencies( self ): '''Verify that each repository can depend on the other without causing an infinite loop.''' filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) @@ -117,12 +105,14 @@ # In this case, the displayed dependency will specify the tip revision, but this will not always be the case. self.check_repository_dependency( filtering_repository, freebayes_repository, self.get_repository_tip( freebayes_repository ) ) self.check_repository_dependency( freebayes_repository, filtering_repository, self.get_repository_tip( filtering_repository ) ) + def test_0035_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) for repository in [ freebayes_repository, filtering_repository ]: self.verify_unchanged_repository_metadata( repository ) + def test_0040_verify_tool_dependencies( self ): '''Verify that freebayes displays tool dependencies.''' repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0050_circular_dependencies_4_levels.py --- a/test/tool_shed/functional/test_0050_circular_dependencies_4_levels.py +++ b/test/tool_shed/functional/test_0050_circular_dependencies_4_levels.py @@ -32,8 +32,10 @@ category_name = 'Test 0050 Circular Dependencies 5 Levels' category_description = 'Test circular dependency features' + class TestRepositoryCircularDependenciesToNLevels( ShedTwillTestCase ): '''Verify that the code correctly handles circular dependencies down to n levels.''' + def test_0000_initiate_users( self ): """Create necessary user accounts.""" self.logout() @@ -46,6 +48,7 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_convert_repository( self ): '''Create and populate convert_chars_0050.''' category = self.create_category( name=category_name, description=category_description ) @@ -66,6 +69,7 @@ commit_message='Uploaded convert_chars tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0010_create_column_repository( self ): '''Create and populate convert_chars_0050.''' category = self.create_category( name=category_name, description=category_description ) @@ -84,6 +88,7 @@ commit_message='Uploaded column_maker tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0015_create_emboss_datatypes_repository( self ): '''Create and populate emboss_datatypes_0050.''' category = self.create_category( name=category_name, description=category_description ) @@ -104,6 +109,7 @@ commit_message='Uploaded datatypes_conf.xml.', strings_displayed=[], strings_not_displayed=[] ) + def test_0020_create_emboss_repository( self ): '''Create and populate emboss_0050.''' category = self.create_category( name=category_name, description=category_description ) @@ -122,20 +128,7 @@ commit_message='Uploaded emboss tarball.', strings_displayed=[], strings_not_displayed=[] ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - repository_dependencies_path = self.generate_temp_path( 'test_0050', additional_paths=[ 'emboss' ] ) - self.generate_repository_dependency_xml( [ datatypes_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Emboss depends on the emboss_datatypes repository.' ) - self.upload_file( repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on emboss_datatypes.', - strings_displayed=[], - strings_not_displayed=[] ) + def test_0025_create_filtering_repository( self ): '''Create and populate filtering_0050.''' category = self.create_category( name=category_name, description=category_description ) @@ -154,20 +147,7 @@ commit_message='Uploaded filtering 1.1.0 tarball.', strings_displayed=[], strings_not_displayed=[] ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - repository_dependencies_path = self.generate_temp_path( 'test_0050', additional_paths=[ 'filtering' ] ) - self.generate_repository_dependency_xml( [ emboss_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Filtering depends on the emboss repository.' ) - self.upload_file( filtering_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on emboss.', - strings_displayed=[], - strings_not_displayed=[] ) + def test_0030_create_freebayes_repository( self ): '''Create and populate freebayes_0050.''' category = self.create_category( name=category_name, description=category_description ) @@ -186,6 +166,7 @@ commit_message='Uploaded freebayes tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0035_create_bismark_repository( self ): '''Create and populate bismark_0050.''' category = self.create_category( name=category_name, description=category_description ) @@ -204,10 +185,11 @@ commit_message='Uploaded bismark tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0040_create_and_upload_dependency_definitions( self ): column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) - emboss_datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) @@ -219,14 +201,21 @@ # emboss_datatypes depends on bismark # freebayes depends on freebayes, emboss, emboss_datatypes, and column_maker # filtering depends on emboss - self.create_repository_dependency( convert_repository, depends_on=[ column_repository ], filepath=dependency_xml_path ) - self.create_repository_dependency( column_repository, depends_on=[ convert_repository ], filepath=dependency_xml_path ) - self.create_repository_dependency( emboss_datatypes_repository, depends_on=[ bismark_repository ], filepath=dependency_xml_path ) - self.create_repository_dependency( emboss_repository, depends_on=[ emboss_datatypes_repository ], filepath=dependency_xml_path ) - self.create_repository_dependency( freebayes_repository, - depends_on=[ freebayes_repository, emboss_datatypes_repository, emboss_repository, column_repository ], + column_tuple = ( self.url, column_repository.name, column_repository.user.username, self.get_repository_tip( column_repository ) ) + convert_tuple = ( self.url, convert_repository.name, convert_repository.user.username, self.get_repository_tip( convert_repository ) ) + freebayes_tuple = ( self.url, freebayes_repository.name, freebayes_repository.user.username, self.get_repository_tip( freebayes_repository ) ) + emboss_tuple = ( self.url, emboss_repository.name, emboss_repository.user.username, self.get_repository_tip( emboss_repository ) ) + datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) + bismark_tuple = ( self.url, bismark_repository.name, bismark_repository.user.username, self.get_repository_tip( bismark_repository ) ) + self.create_repository_dependency( repository=convert_repository, repository_tuples=[ column_tuple ], filepath=dependency_xml_path ) + self.create_repository_dependency( repository=column_repository, repository_tuples=[ convert_tuple ], filepath=dependency_xml_path ) + self.create_repository_dependency( repository=datatypes_repository, repository_tuples=[ bismark_tuple ], filepath=dependency_xml_path ) + self.create_repository_dependency( repository=emboss_repository, repository_tuples=[ datatypes_tuple ], filepath=dependency_xml_path ) + self.create_repository_dependency( repository=freebayes_repository, + repository_tuples=[ freebayes_tuple, datatypes_tuple, emboss_tuple, column_tuple ], filepath=dependency_xml_path ) - self.create_repository_dependency( filtering_repository, depends_on=[ emboss_repository ], filepath=dependency_xml_path ) + self.create_repository_dependency( repository=filtering_repository, repository_tuples=[ emboss_tuple ], filepath=dependency_xml_path ) + def test_0045_verify_repository_dependencies( self ): '''Verify that the generated dependency circle does not cause an infinite loop. Expected structure: @@ -264,6 +253,7 @@ strings_displayed = [ 'freebayes_0050 depends on freebayes_0050, emboss_datatypes_0050, emboss_0050, column_maker_0050.' ] self.display_manage_repository_page( freebayes_repository, strings_displayed=strings_displayed ) + def test_0050_verify_tool_dependencies( self ): '''Check that freebayes and emboss display tool dependencies.''' freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) @@ -271,6 +261,7 @@ self.display_manage_repository_page( freebayes_repository, strings_displayed=[ 'freebayes', '0.9.4_9696d0ce8a9', 'samtools', '0.1.18', 'Tool dependencies', 'package' ] ) self.display_manage_repository_page( emboss_repository, strings_displayed=[ 'Tool dependencies', 'emboss', '5.0.0', 'package' ] ) + def test_0055_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' emboss_datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0080_advanced_circular_dependencies.py --- a/test/tool_shed/functional/test_0080_advanced_circular_dependencies.py +++ b/test/tool_shed/functional/test_0080_advanced_circular_dependencies.py @@ -12,8 +12,10 @@ category_name = 'Test 0080 Advanced Circular Dependencies' category_description = 'Test circular dependency features' + class TestRepositoryCircularDependencies( ShedTwillTestCase ): '''Verify that the code correctly handles circular dependencies.''' + def test_0000_initiate_users( self ): """Create necessary user accounts.""" self.logout() @@ -26,6 +28,7 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_column_repository( self ): """Create and populate the column_maker repository.""" category = self.create_category( name=category_name, description=category_description ) @@ -46,6 +49,7 @@ commit_message='Uploaded column_maker tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0005_create_convert_repository( self ): """Create and populate the convert_chars repository.""" self.logout() @@ -68,46 +72,30 @@ commit_message='Uploaded convert_chars tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0020_create_repository_dependencies( self ): - '''Upload a repository_dependencies.xml file that specifies the current revision of freebayes to the filtering_0040 repository.''' + '''Upload a repository_dependencies.xml file that specifies the current revision of convert_chars_0080 to the column_maker_0080 repository.''' convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0080', additional_paths=[ 'convert' ] ) - self.generate_repository_dependency_xml( [ convert_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Column maker depends on the convert repository.' ) - self.upload_file( column_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=True, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on convert_chars.', - strings_displayed=[], - strings_not_displayed=[] ) + repository_tuple = ( self.url, convert_repository.name, convert_repository.user.username, self.get_repository_tip( convert_repository ) ) + self.create_repository_dependency( repository=column_repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) + def test_0025_create_dependency_on_filtering( self ): '''Upload a repository_dependencies.xml file that specifies the current revision of filtering to the freebayes_0040 repository.''' convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0080', additional_paths=[ 'convert' ] ) - self.generate_repository_dependency_xml( [ column_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Convert chars depends on the column_maker repository.' ) - self.upload_file( convert_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=True, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on column_maker.', - strings_displayed=[], - strings_not_displayed=[] ) + repository_tuple = ( self.url, column_repository.name, column_repository.user.username, self.get_repository_tip( column_repository ) ) + self.create_repository_dependency( repository=convert_repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) + def test_0030_verify_repository_dependencies( self ): '''Verify that each repository can depend on the other without causing an infinite loop.''' convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) self.check_repository_dependency( convert_repository, column_repository, self.get_repository_tip( column_repository ) ) self.check_repository_dependency( column_repository, convert_repository, self.get_repository_tip( convert_repository ) ) + def test_0035_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0090_tool_search.py --- a/test/tool_shed/functional/test_0090_tool_search.py +++ b/test/tool_shed/functional/test_0090_tool_search.py @@ -28,8 +28,10 @@ category_name = 'Test 0090 Tool Search And Installation' category_description = 'Test 0090 Tool Search And Installation' + class TestRepositoryCircularDependenciesAgain( ShedTwillTestCase ): '''Test more features related to repository dependencies.''' + def test_0000_initiate_users( self ): """Create necessary user accounts.""" self.logout() @@ -42,6 +44,7 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_bwa_base_repository( self ): '''Create and populate bwa_base_0090.''' category = self.create_category( name=category_name, description=category_description ) @@ -62,6 +65,7 @@ commit_message='Uploaded BWA tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0010_create_bwa_color_repository( self ): '''Create and populate bwa_color_0090.''' category = self.create_category( name=category_name, description=category_description ) @@ -82,6 +86,7 @@ commit_message='Uploaded BWA color tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0015_create_emboss_datatypes_repository( self ): '''Create and populate emboss_datatypes_0090.''' category = self.create_category( name=category_name, description=category_description ) @@ -102,6 +107,7 @@ commit_message='Uploaded datatypes_conf.xml.', strings_displayed=[], strings_not_displayed=[] ) + def test_0020_create_emboss_repository( self ): '''Create and populate emboss_0090.''' category = self.create_category( name=category_name, description=category_description ) @@ -120,20 +126,7 @@ commit_message='Uploaded emboss tarball.', strings_displayed=[], strings_not_displayed=[] ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - repository_dependencies_path = self.generate_temp_path( 'test_0090', additional_paths=[ 'emboss' ] ) - self.generate_repository_dependency_xml( [ datatypes_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Emboss depends on the emboss_datatypes repository.' ) - self.upload_file( repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=True, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on emboss_datatypes.', - strings_displayed=[], - strings_not_displayed=[] ) + def test_0025_create_filtering_repository( self ): '''Create and populate filtering_0090.''' category = self.create_category( name=category_name, description=category_description ) @@ -152,20 +145,7 @@ commit_message='Uploaded filtering 1.1.0 tarball.', strings_displayed=[], strings_not_displayed=[] ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - repository_dependencies_path = self.generate_temp_path( 'test_0090', additional_paths=[ 'filtering' ] ) - self.generate_repository_dependency_xml( [ emboss_repository ], - self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), - dependency_description='Filtering depends on the emboss repository.' ) - self.upload_file( filtering_repository, - filename='repository_dependencies.xml', - filepath=repository_dependencies_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on emboss.', - strings_displayed=[], - strings_not_displayed=[] ) + def test_0030_create_freebayes_repository( self ): '''Create and populate freebayes_0090.''' category = self.create_category( name=category_name, description=category_description ) @@ -184,19 +164,25 @@ commit_message='Uploaded freebayes tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0035_create_and_upload_dependency_definitions( self ): '''Create and upload repository dependency definitions.''' bwa_color_repository = test_db_util.get_repository_by_name_and_owner( bwa_color_repository_name, common.test_user_1_name ) bwa_base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - emboss_datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) dependency_xml_path = self.generate_temp_path( 'test_0090', additional_paths=[ 'freebayes' ] ) - self.create_repository_dependency( emboss_repository, depends_on=[ emboss_datatypes_repository ], filepath=dependency_xml_path ) - self.create_repository_dependency( filtering_repository, depends_on=[ freebayes_repository ], filepath=dependency_xml_path ) - self.create_repository_dependency( bwa_base_repository, depends_on=[ emboss_repository ], filepath=dependency_xml_path ) - self.create_repository_dependency( bwa_color_repository, depends_on=[ filtering_repository ], filepath=dependency_xml_path ) + freebayes_tuple = ( self.url, freebayes_repository.name, freebayes_repository.user.username, self.get_repository_tip( freebayes_repository ) ) + emboss_tuple = ( self.url, emboss_repository.name, emboss_repository.user.username, self.get_repository_tip( emboss_repository ) ) + datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) + filtering_tuple = ( self.url, filtering_repository.name, filtering_repository.user.username, self.get_repository_tip( filtering_repository ) ) + self.create_repository_dependency( repository=emboss_repository, repository_tuples=[ datatypes_tuple ], filepath=dependency_xml_path ) + self.create_repository_dependency( repository=filtering_repository, repository_tuples=[ freebayes_tuple ], filepath=dependency_xml_path ) + self.create_repository_dependency( repository=bwa_base_repository, repository_tuples=[ emboss_tuple ], filepath=dependency_xml_path ) + self.create_repository_dependency( repository=bwa_color_repository, repository_tuples=[ filtering_tuple ], filepath=dependency_xml_path ) + def test_0040_verify_repository_dependencies( self ): '''Verify the generated dependency structure.''' bwa_color_repository = test_db_util.get_repository_by_name_and_owner( bwa_color_repository_name, common.test_user_1_name ) diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0100_complex_repository_dependencies.py --- a/test/tool_shed/functional/test_0100_complex_repository_dependencies.py +++ b/test/tool_shed/functional/test_0100_complex_repository_dependencies.py @@ -1,6 +1,9 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os import tool_shed.base.test_db_util as test_db_util +import logging +log = logging.getLogger( __name__ ) + bwa_base_repository_name = 'bwa_base_repository_0100' bwa_base_repository_description = "BWA Base" bwa_base_repository_long_description = "BWA tool that depends on bwa 0.5.9, with a complex repository dependency pointing at bwa_tool_repository_0100" @@ -12,8 +15,10 @@ category_name = 'Test 0100 Complex Repository Dependencies' category_description = 'Test 0100 Complex Repository Dependencies' + class TestComplexRepositoryDependencies( ShedTwillTestCase ): '''Test features related to complex repository dependencies.''' + def test_0000_initiate_users( self ): """Create necessary user accounts.""" self.logout() @@ -26,6 +31,7 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_bwa_tool_repository( self ): '''Create and populate bwa_tool_repository_0100.''' category = self.create_category( name=category_name, description=category_description ) @@ -49,6 +55,7 @@ strings_not_displayed=[] ) # Visit the manage repository page for bwa_tool_repository_0100. self.display_manage_repository_page( repository, strings_displayed=[ 'Tool dependencies', 'may not be', 'in this repository' ] ) + def test_0010_create_bwa_base_repository( self ): '''Create and populate bwa_base_0100.''' category = self.create_category( name=category_name, description=category_description ) @@ -71,30 +78,30 @@ commit_message='Uploaded bwa_base.tar with tool wrapper XML, but without tool dependency XML.', strings_displayed=[], strings_not_displayed=[] ) + def test_0015_generate_complex_repository_dependency_invalid_shed_url( self ): '''Generate and upload a complex repository definition that specifies an invalid tool shed URL.''' dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'invalid' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The repository named bwa_base_repository_0100 is the dependent repository. - repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) # The tool_repository named bwa_tool_repository_0100 is the required repository. tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) url = 'http://http://this is not an url!' name = 'bwa_tool_repository_0100' owner = 'user1' changeset_revision = self.get_repository_tip( tool_repository ) - self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) strings_displayed = [ 'Repository dependencies are currently supported only within the same tool shed' ] # Populate the dependent repository named bwa_base_repository_0100 with an invalid tool_dependencies.xml file. - self.upload_file( repository, - filename='tool_dependencies.xml', - filepath=dependency_path, - valid_tools_only=False, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on bwa_tool_0100 with invalid url.', - strings_displayed=strings_displayed, - strings_not_displayed=[] ) + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=base_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + strings_displayed=strings_displayed, + complex=True, + package='bwa', + version='0.5.9' ) + def test_0020_generate_complex_repository_dependency_invalid_repository_name( self ): '''Generate and upload a complex repository definition that specifies an invalid repository name.''' dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'invalid' ] ) @@ -107,18 +114,17 @@ name = 'invalid_repository!?' owner = 'user1' changeset_revision = self.get_repository_tip( tool_repository ) - self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) strings_displayed = [ 'because the name is invalid' ] - # # Populate the dependent base_repository named bwa_tool_repository_0100 with an invalid tool_dependencies.xml file. - self.upload_file( base_repository, - filename='tool_dependencies.xml', - filepath=dependency_path, - valid_tools_only=False, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on bwa_tool_0100 with invalid repository name.', - strings_displayed=strings_displayed, - strings_not_displayed=[] ) + # Populate the dependent base_repository named bwa_tool_repository_0100 with an invalid tool_dependencies.xml file. + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=base_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + strings_displayed=strings_displayed, + complex=True, + package='bwa', + version='0.5.9' ) + def test_0025_generate_complex_repository_dependency_invalid_owner_name( self ): '''Generate and upload a complex repository definition that specifies an invalid owner.''' dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'invalid' ] ) @@ -131,17 +137,16 @@ name = 'bwa_tool_repository_0100' owner = 'invalid_owner!?' changeset_revision = self.get_repository_tip( tool_repository ) - self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'because the owner is invalid.' ] - self.upload_file( base_repository, - filename='tool_dependencies.xml', - filepath=dependency_path, - valid_tools_only=False, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on bwa_tool_0100 with invalid owner.', - strings_displayed=strings_displayed, - strings_not_displayed=[] ) + strings_displayed = [ 'because the owner is invalid.' ] + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=base_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + strings_displayed=strings_displayed, + complex=True, + package='bwa', + version='0.5.9' ) + def test_0030_generate_complex_repository_dependency_invalid_changeset_revision( self ): '''Generate and upload a complex repository definition that specifies an invalid changeset revision.''' dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'invalid' ] ) @@ -154,17 +159,16 @@ name = 'bwa_tool_repository_0100' owner = 'user1' changeset_revision = '1234abcd' - self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) strings_displayed = [ 'because the changeset revision is invalid.' ] - self.upload_file( base_repository, - filename='tool_dependencies.xml', - filepath=dependency_path, - valid_tools_only=False, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on bwa_tool_0100 with invalid changeset revision.', - strings_displayed=strings_displayed, - strings_not_displayed=[] ) + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=base_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + strings_displayed=strings_displayed, + complex=True, + package='bwa', + version='0.5.9' ) + def test_0035_generate_complex_repository_dependency( self ): '''Generate and upload a valid tool_dependencies.xml file that specifies bwa_tool_repository_0100.''' # The base_repository named bwa_base_repository_0100 is the dependent repository. @@ -172,25 +176,20 @@ # The tool_repository named bwa_tool_repository_0100 is the required repository. tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_tool_repository_name, common.test_user_1_name ) dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex' ] ) - xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) url = self.url name = 'bwa_tool_repository_0100' owner = 'user1' changeset_revision = self.get_repository_tip( tool_repository ) - self.generate_repository_dependency_xml( [ tool_repository ], xml_filename, complex=True, package='bwa', version='0.5.9' ) - # Upload the valid tool_dependencies.xml file to bwa_base_repository_0100 that specifies bwa_tool_repository_0100 - # as a repository dependency via a complex repository dependency definition. - self.upload_file( base_repository, - filename='tool_dependencies.xml', - filepath=dependency_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded valid complex dependency on bwa_tool_0100.', - strings_displayed=[], - strings_not_displayed=[] ) - self.check_repository_dependency( base_repository, tool_repository ) - self.display_manage_repository_page( base_repository, strings_displayed=[ 'bwa', '0.5.9', 'package' ] ) + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=base_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + complex=True, + package='bwa', + version='0.5.9' ) + self.check_repository_dependency( base_repository, depends_on_repository=tool_repository ) + self.display_manage_repository_page( base_repository, strings_displayed=[ 'bwa', '0.5.9', 'package', changeset_revision ] ) + def test_0040_generate_tool_dependency( self ): '''Generate and upload a new tool_dependencies.xml file that specifies an arbitrary file on the filesystem, and verify that bwa_base depends on the new changeset revision.''' # The base_repository named bwa_base_repository_0100 is the dependent repository. diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py --- a/test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py +++ b/test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py @@ -12,8 +12,10 @@ category_name = 'Test 0110 Invalid Repository Dependencies' category_desc = 'Test 0110 Invalid Repository Dependencies' + class TestBasicRepositoryDependencies( ShedTwillTestCase ): '''Testing emboss 5 with repository dependencies.''' + def test_0000_initiate_users( self ): """Create necessary user accounts and login as an admin user.""" self.logout() @@ -26,9 +28,11 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_category( self ): """Create a category for this test suite""" self.create_category( name=category_name, description=category_desc ) + def test_0010_create_emboss_datatypes_repository_and_upload_tarball( self ): '''Create and populate the emboss_datatypes repository.''' self.logout() @@ -49,10 +53,12 @@ commit_message='Uploaded datatypes_conf.xml.', strings_displayed=[], strings_not_displayed=[] ) + def test_0015_verify_datatypes_in_datatypes_repository( self ): '''Verify that the emboss_datatypes repository contains datatype entries.''' repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) self.display_manage_repository_page( repository, strings_displayed=[ 'Datatypes', 'equicktandem', 'hennig86', 'vectorstrip' ] ) + def test_0020_create_emboss_5_repository_and_upload_files( self ): '''Create and populate the emboss_5_0110 repository.''' category = test_db_util.get_category_by_name( category_name ) @@ -71,27 +77,25 @@ commit_message='Uploaded emboss tool tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0025_generate_repository_dependency_with_invalid_url( self ): '''Generate a repository dependency for emboss 5 with an invalid URL.''' dependency_path = self.generate_temp_path( 'test_0110', additional_paths=[ 'simple' ] ) xml_filename = self.get_filename( 'repository_dependencies.xml', filepath=dependency_path ) - repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) url = 'http://http://this is not an url!' - name = repository.name - owner = repository.user.username - changeset_revision = self.get_repository_tip( repository ) - self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) + name = datatypes_repository.name + owner = datatypes_repository.user.username + changeset_revision = self.get_repository_tip( datatypes_repository ) strings_displayed = [ 'Repository dependencies are currently supported only within the same tool shed' ] - self.upload_file( emboss_repository, - filename='repository_dependencies.xml', - filepath=dependency_path, - valid_tools_only=False, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on emboss_datatypes_0110 with invalid url.', - strings_displayed=strings_displayed, - strings_not_displayed=[] ) + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=emboss_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + strings_displayed=strings_displayed, + complex=False ) + def test_0030_generate_repository_dependency_with_invalid_name( self ): '''Generate a repository dependency for emboss 5 with an invalid name.''' dependency_path = self.generate_temp_path( 'test_0110', additional_paths=[ 'simple' ] ) @@ -102,17 +106,14 @@ name = '!?invalid?!' owner = repository.user.username changeset_revision = self.get_repository_tip( repository ) - self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) strings_displayed = [ 'because the name is invalid.' ] - self.upload_file( emboss_repository, - filename='repository_dependencies.xml', - filepath=dependency_path, - valid_tools_only=False, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on emboss_datatypes_0110 with invalid name.', - strings_displayed=strings_displayed, - strings_not_displayed=[] ) + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=emboss_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + strings_displayed=strings_displayed, + complex=False ) + def test_0035_generate_repository_dependency_with_invalid_owner( self ): '''Generate a repository dependency for emboss 5 with an invalid owner.''' dependency_path = self.generate_temp_path( 'test_0110', additional_paths=[ 'simple' ] ) @@ -123,17 +124,14 @@ name = repository.name owner = '!?invalid?!' changeset_revision = self.get_repository_tip( repository ) - self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) strings_displayed = [ 'because the owner is invalid.' ] - self.upload_file( emboss_repository, - filename='repository_dependencies.xml', - filepath=dependency_path, - valid_tools_only=False, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on emboss_datatypes_0110 with invalid owner.', - strings_displayed=strings_displayed, - strings_not_displayed=[] ) + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=emboss_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + strings_displayed=strings_displayed, + complex=False ) + def test_0040_generate_repository_dependency_with_invalid_changeset_revision( self ): '''Generate a repository dependency for emboss 5 with an invalid changeset revision.''' dependency_path = self.generate_temp_path( 'test_0110', additional_paths=[ 'simple', 'invalid' ] ) @@ -144,14 +142,10 @@ name = repository.name owner = repository.user.username changeset_revision = '!?invalid?!' - self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) strings_displayed = [ 'because the changeset revision is invalid.' ] - self.upload_file( emboss_repository, - filename='repository_dependencies.xml', - filepath=dependency_path, - valid_tools_only=False, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message='Uploaded dependency on emboss_datatypes_0110 with invalid changeset revision.', - strings_displayed=strings_displayed, - strings_not_displayed=[] ) + repository_tuple = ( url, name, owner, changeset_revision ) + self.create_repository_dependency( repository=emboss_repository, + filepath=dependency_path, + repository_tuples=[ repository_tuple ], + strings_displayed=strings_displayed, + complex=False ) diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py --- a/test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py +++ b/test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py @@ -23,7 +23,9 @@ base_datatypes_count = 0 repository_datatypes_count = 0 + class TestRepositoryMultipleOwners( ShedTwillTestCase ): + def test_0000_initiate_users( self ): """Create necessary user accounts and login as an admin user.""" """ @@ -45,6 +47,7 @@ admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) + def test_0005_create_datatypes_repository( self ): """Create and populate the blast_datatypes_0120 repository""" """ @@ -71,6 +74,7 @@ commit_message='Uploaded blast_datatypes tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0010_verify_datatypes_repository( self ): '''Verify the blast_datatypes_0120 repository.''' ''' @@ -83,6 +87,7 @@ strings_displayed = [ 'BlastXml', 'BlastNucDb', 'BlastProtDb', 'application/xml', 'text/html', 'blastxml', 'blastdbn', 'blastdbp'] self.display_manage_repository_page( repository, strings_displayed=strings_displayed ) repository_datatypes_count = int( self.get_repository_datatypes_count( repository ) ) + def test_0015_create_tool_repository( self ): """Create and populate the blastxml_to_top_descr_0120 repository""" """ @@ -109,6 +114,7 @@ commit_message='Uploaded blastxml_to_top_descr tarball.', strings_displayed=[], strings_not_displayed=[] ) + def test_0020_verify_tool_repository( self ): '''Verify the blastxml_to_top_descr_0120 repository.''' ''' @@ -119,6 +125,7 @@ strings_displayed = [ 'blastxml_to_top_descr_0120', 'BLAST top hit descriptions', 'Make a table from BLAST XML' ] strings_displayed.extend( [ '0.0.1', 'Valid tools'] ) self.display_manage_repository_page( repository, strings_displayed=strings_displayed ) + def test_0025_create_repository_dependency( self ): '''Create a repository dependency on blast_datatypes_0120.''' ''' @@ -128,7 +135,9 @@ datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_2_name ) tool_repository = test_db_util.get_repository_by_name_and_owner( tool_repository_name, common.test_user_1_name ) dependency_xml_path = self.generate_temp_path( 'test_0120', additional_paths=[ 'dependencies' ] ) - self.create_repository_dependency( repository=tool_repository, depends_on=[ datatypes_repository ], filepath=dependency_xml_path ) + datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) + self.create_repository_dependency( repository=tool_repository, repository_tuples=[ datatypes_tuple ], filepath=dependency_xml_path ) + def test_0040_verify_repository_dependency( self ): '''Verify the created repository dependency.''' ''' diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0150_prior_installation_required.py --- /dev/null +++ b/test/tool_shed/functional/test_0150_prior_installation_required.py @@ -0,0 +1,109 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +column_repository_name = 'column_maker_0150' +column_repository_description = "Add column" +column_repository_long_description = "Compute an expression on every row" + +convert_repository_name = 'convert_chars_0150' +convert_repository_description = "Convert delimiters" +convert_repository_long_description = "Convert delimiters to tab" + +category_name = 'Test 0150 Simple Prior Installation' +category_description = 'Test 0150 Simple Prior Installation' + +''' +Create column_maker and convert_chars. + +Column maker repository dependency: +<repository toolshed="self.url" name="convert_chars" owner="test" changeset_revision="c3041382815c" prior_installation_required="True" /> + +Verify display. + +Galaxy side: + +Install column_maker. +Verify that convert_chars was installed first, contrary to the ordering that would be present without prior_installation_required. +''' + + +class TestSimplePriorInstallation( ShedTwillTestCase ): + '''Test features related to datatype converters.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts.""" + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + + def test_0005_create_convert_repository( self ): + '''Create and populate convert_chars_0150.''' + category = self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + repository = self.get_or_create_repository( name=convert_repository_name, + description=convert_repository_description, + long_description=convert_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='convert_chars/convert_chars.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded convert_chars tarball.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0010_create_column_repository( self ): + '''Create and populate convert_chars_0150.''' + category = self.create_category( name=category_name, description=category_description ) + repository = self.get_or_create_repository( name=column_repository_name, + description=column_repository_description, + long_description=column_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='column_maker/column_maker.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded column_maker tarball.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0015_create_repository_dependency( self ): + '''Create a repository dependency specifying convert_chars.''' + ''' + Column maker repository dependency: + <repository toolshed="self.url" name="convert_chars" owner="test" changeset_revision="<tip>" prior_installation_required="True" /> + ''' + column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + dependency_xml_path = self.generate_temp_path( 'test_0150', additional_paths=[ 'column' ] ) + convert_tuple = ( self.url, convert_repository.name, convert_repository.user.username, self.get_repository_tip( convert_repository ) ) + self.create_repository_dependency( repository=column_repository, + repository_tuples=[ convert_tuple ], + filepath=dependency_xml_path, + prior_installation_required=True ) + + def test_0020_verify_repository_dependency( self ): + '''Verify that the previously generated repositiory dependency displays correctly.''' + column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + self.check_repository_dependency( repository=column_repository, + depends_on_repository=convert_repository, + depends_on_changeset_revision=None, + changeset_revision=None ) + diff -r 2c50935e70c8a483bcd09b358765408543a153fb -r 8d72c9adccf92f8a1971ec7d9b622de43cbd6f48 test/tool_shed/functional/test_0160_circular_prior_installation_required.py --- /dev/null +++ b/test/tool_shed/functional/test_0160_circular_prior_installation_required.py @@ -0,0 +1,161 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +filter_repository_name = 'filtering_0160' +filter_repository_description = "Galaxy's filtering tool for test 0160" +filter_repository_long_description = "Long description of Galaxy's filtering tool for test 0160" + +column_repository_name = 'column_maker_0160' +column_repository_description = "Add column" +column_repository_long_description = "Compute an expression on every row" + +convert_repository_name = 'convert_chars_0160' +convert_repository_description = "Convert delimiters" +convert_repository_long_description = "Convert delimiters to tab" + +category_name = 'Test 0160 Simple Prior Installation' +category_description = 'Test 0160 Simple Prior Installation' + +''' +Create column_maker and convert_chars. + +Column maker repository dependency: +<repository toolshed="self.url" name="convert_chars" owner="test" changeset_revision="c3041382815c" prior_installation_required="True" /> + +Verify display. +''' + + +class TestSimplePriorInstallation( ShedTwillTestCase ): + '''Test features related to datatype converters.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts.""" + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + + def test_0005_create_convert_repository( self ): + '''Create and populate convert_chars_0160.''' + category = self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + repository = self.get_or_create_repository( name=convert_repository_name, + description=convert_repository_description, + long_description=convert_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='convert_chars/convert_chars.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded convert_chars tarball.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0010_create_column_repository( self ): + '''Create and populate convert_chars_0160.''' + category = self.create_category( name=category_name, description=category_description ) + repository = self.get_or_create_repository( name=column_repository_name, + description=column_repository_description, + long_description=column_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='column_maker/column_maker.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded column_maker tarball.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0015_create_filtering_repository( self ): + '''Create and populate filtering_0160.''' + category = self.create_category( name=category_name, description=category_description ) + repository = self.get_or_create_repository( name=filter_repository_name, + description=filter_repository_description, + long_description=filter_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='filtering/filtering_1.1.0.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=True, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded filtering 1.1.0 tarball.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0020_create_repository_dependency( self ): + '''Create a repository dependency specifying convert_chars.''' + ''' + Each of the three repositories should depend on the other two, to make this as circular as possible. + ''' + filter_repository = test_db_util.get_repository_by_name_and_owner( filter_repository_name, common.test_user_1_name ) + column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + dependency_xml_path = self.generate_temp_path( 'test_0160', additional_paths=[ 'column' ] ) + filter_revision = self.get_repository_tip( filter_repository ) + column_revision = self.get_repository_tip( column_repository ) + convert_revision = self.get_repository_tip( convert_repository ) + column_tuple = ( self.url, column_repository.name, column_repository.user.username, column_revision ) + convert_tuple = ( self.url, convert_repository.name, convert_repository.user.username, convert_revision ) + filter_tuple = ( self.url, filter_repository.name, filter_repository.user.username, filter_revision ) + self.create_repository_dependency( repository=column_repository, + repository_tuples=[ convert_tuple, filter_tuple], + filepath=dependency_xml_path, + prior_installation_required=False ) + self.create_repository_dependency( repository=convert_repository, + repository_tuples=[ column_tuple, filter_tuple ], + filepath=dependency_xml_path, + prior_installation_required=False ) + self.create_repository_dependency( repository=filter_repository, + repository_tuples=[ convert_tuple, column_tuple ], + filepath=dependency_xml_path, + prior_installation_required=True ) + + def test_0025_verify_repository_dependency( self ): + '''Verify that the previously generated repositiory dependency displays correctly.''' + filter_repository = test_db_util.get_repository_by_name_and_owner( filter_repository_name, common.test_user_1_name ) + column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + self.check_repository_dependency( repository=column_repository, + depends_on_repository=convert_repository, + depends_on_changeset_revision=None, + changeset_revision=None ) + self.check_repository_dependency( repository=column_repository, + depends_on_repository=filter_repository, + depends_on_changeset_revision=None, + changeset_revision=None ) + self.check_repository_dependency( repository=convert_repository, + depends_on_repository=column_repository, + depends_on_changeset_revision=None, + changeset_revision=None ) + self.check_repository_dependency( repository=convert_repository, + depends_on_repository=filter_repository, + depends_on_changeset_revision=None, + changeset_revision=None ) + self.check_repository_dependency( repository=filter_repository, + depends_on_repository=column_repository, + depends_on_changeset_revision=None, + changeset_revision=None ) + self.check_repository_dependency( repository=filter_repository, + depends_on_repository=convert_repository, + depends_on_changeset_revision=None, + changeset_revision=None ) + This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0