- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: jgoecks: Trackster: more name location fixes.
by commits-noreply＠bitbucket.org 18 Oct '13

18 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/503d939099fc/ Changeset: 503d939099fc User: jgoecks Date: 2013-10-18 04:45:23 Summary: Trackster: more name location fixes. Affected #: 1 file diff -r f0875c78b7864c5ac75be62e9a52611bfa2a6cf0 -r 503d939099fcd8913202300b83221a8d3269b9f9 static/scripts/viz/trackster_ui.js --- a/static/scripts/viz/trackster_ui.js +++ b/static/scripts/viz/trackster_ui.js @@ -75,7 +75,7 @@ }); // FIXME: give unique IDs to Drawables and save overview as ID. - var overview_track_name = (view.overview_drawable ? view.overview_drawable.name : null), + var overview_track_name = (view.overview_drawable ? view.overview_drawable.prefs.name : null), viz_config = { 'view': view.to_dict(), 'viewport': { 'chrom': view.chrom, 'start': view.low , 'end': view.high, 'overview': overview_track_name }, @@ -88,7 +88,7 @@ dataType: "json", data: { 'id' : view.vis_id, - 'title' : view.name, + 'title' : view.prefs.name, 'dbkey' : view.dbkey, 'type' : 'trackster', 'vis_json' : JSON.stringify(viz_config) @@ -265,7 +265,7 @@ // Set overview. var overview_drawable; for (var i = 0; i < view.drawables.length; i++) { - if (view.drawables[i].name === overview_drawable_name) { + if (view.drawables[i].prefs.name === overview_drawable_name) { view.set_overview(view.drawables[i]); break; } Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Set visualization naming to refer to new location.
by commits-noreply＠bitbucket.org 18 Oct '13

18 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f0875c78b786/ Changeset: f0875c78b786 User: jgoecks Date: 2013-10-18 04:22:36 Summary: Set visualization naming to refer to new location. Affected #: 1 file diff -r 0d46133c90452dc8815de7057b13548f06beb7bb -r f0875c78b7864c5ac75be62e9a52611bfa2a6cf0 static/scripts/viz/trackster.js --- a/static/scripts/viz/trackster.js +++ b/static/scripts/viz/trackster.js @@ -197,7 +197,7 @@ init_editor : function () { // set title - $("#center .unified-panel-title").text(view.name + " (" + view.dbkey + ")"); + $("#center .unified-panel-title").text(view.prefs.name + " (" + view.dbkey + ")"); // add dataset if (galaxy_config.app.add_dataset) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Add smpt_ssl parameter to force SSL SMTP connection from start.
by commits-noreply＠bitbucket.org 18 Oct '13

18 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0d46133c9045/ Changeset: 0d46133c9045 User: jmchilton Date: 2013-10-18 01:09:55 Summary: Add smpt_ssl parameter to force SSL SMTP connection from start. This is based on a slightly reworked commit from @aswarren (pull request 198). I tweaked it by dropping smtp_from (as discussed in pull request), and fixing a small string versus bool bug. Tested both ways and it seems to work, with any luck though there is a bug in there somewhere and Galaxy admins everywhere will get a break until it is discovered :). Affected #: 3 files diff -r 90547efa211fb9c6480662129591ab28acbc0ffe -r 0d46133c90452dc8815de7057b13548f06beb7bb lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -159,6 +159,7 @@ self.smtp_server = kwargs.get( 'smtp_server', None ) self.smtp_username = kwargs.get( 'smtp_username', None ) self.smtp_password = kwargs.get( 'smtp_password', None ) + self.smtp_ssl = kwargs.get( 'smtp_ssl', None ) self.track_jobs_in_database = kwargs.get( 'track_jobs_in_database', 'None' ) self.start_job_runners = listify(kwargs.get( 'start_job_runners', '' )) self.expose_dataset_path = string_as_bool( kwargs.get( 'expose_dataset_path', 'False' ) ) diff -r 90547efa211fb9c6480662129591ab28acbc0ffe -r 0d46133c90452dc8815de7057b13548f06beb7bb lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -873,19 +873,24 @@ log.error( "Mail is not configured for this Galaxy instance." ) log.info( msg ) return - s = smtplib.SMTP() + smtp_ssl = asbool( config.smtp_ssl ) + if smtp_ssl: + s = smtplib.SMTP_SSL() + else: + s = smtplib.SMTP() s.connect( config.smtp_server ) - try: - s.starttls() - log.debug( 'Initiated SSL/TLS connection to SMTP server: %s' % config.smtp_server ) - except RuntimeError, e: - log.warning( 'SSL/TLS support is not available to your Python interpreter: %s' % e ) - except smtplib.SMTPHeloError, e: - log.error( "The server didn't reply properly to the HELO greeting: %s" % e ) - s.close() - raise - except smtplib.SMTPException, e: - log.warning( 'The server does not support the STARTTLS extension: %s' % e ) + if not smtp_ssl: + try: + s.starttls() + log.debug( 'Initiated SSL/TLS connection to SMTP server: %s' % config.smtp_server ) + except RuntimeError, e: + log.warning( 'SSL/TLS support is not available to your Python interpreter: %s' % e ) + except smtplib.SMTPHeloError, e: + log.error( "The server didn't reply properly to the HELO greeting: %s" % e ) + s.close() + raise + except smtplib.SMTPException, e: + log.warning( 'The server does not support the STARTTLS extension: %s' % e ) if config.smtp_username and config.smtp_password: try: s.login( config.smtp_username, config.smtp_password ) diff -r 90547efa211fb9c6480662129591ab28acbc0ffe -r 0d46133c90452dc8815de7057b13548f06beb7bb universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -263,6 +263,9 @@ #smtp_username = None #smtp_password = None +# If your SMTP server requires SSL from the beginning of the connection +# smtp_ssl= False + # On the user registration form, users may choose to join the mailing list. # This is the address of the list they'll be subscribed to. #mailing_join_addr = galaxy-announce-join(a)bx.psu.edu Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 18 Oct '13

18 Oct '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0b440d5c1667/ Changeset: 0b440d5c1667 User: nsoranzo Date: 2013-10-17 12:46:13 Summary: Fix for "NotFound: cannot find 'cov_cutoff' while searching for 'expected.cov_cutoff'" Affected #: 1 file diff -r 329ea7a83af4f389a7c95ee4559d88c6fec0211b -r 0b440d5c1667b06e3a1eb3c8131b925e6da173fd tools/sr_assembly/velvetg.xml --- a/tools/sr_assembly/velvetg.xml +++ b/tools/sr_assembly/velvetg.xml @@ -19,7 +19,7 @@ #if $expected.coverage == "auto": -exp_cov auto #elif $expected.coverage == "value": - -exp_cov $expected.cov_cutoff + -exp_cov $expected.exp_cov #end if #if $contig_lgth.use_contig_lgth == "yes": -min_contig_lgth $contig_lgth.min_contig_lgth https://bitbucket.org/galaxy/galaxy-central/commits/90547efa211f/ Changeset: 90547efa211f User: jmchilton Date: 2013-10-18 00:27:23 Summary: Merge. Affected #: 1 file diff -r 0282484e4c323e1baf79d05813ea8213239a8b4e -r 90547efa211fb9c6480662129591ab28acbc0ffe tools/sr_assembly/velvetg.xml --- a/tools/sr_assembly/velvetg.xml +++ b/tools/sr_assembly/velvetg.xml @@ -19,7 +19,7 @@ #if $expected.coverage == "auto": -exp_cov auto #elif $expected.coverage == "value": - -exp_cov $expected.cov_cutoff + -exp_cov $expected.exp_cov #end if #if $contig_lgth.use_contig_lgth == "yes": -min_contig_lgth $contig_lgth.min_contig_lgth Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: BjoernGruening: If you have a repeat tag and want to get the filetype from the first element you can't access the first element in a from_param type.
by commits-noreply＠bitbucket.org 18 Oct '13

18 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0282484e4c32/ Changeset: 0282484e4c32 User: BjoernGruening Date: 2013-08-14 19:07:31 Summary: If you have a repeat tag and want to get the filetype from the first element you can't access the first element in a from_param type. That commit creates some artifical parameter called 'first' that you can use as value in param_attribute. For example: <option type="from_param" name="singlePaired.mate_list" param_attribute="first.input_mate1.ext" /> I use that to retrive the filetype of paired end reads that are inserted in a repeat tag. Assuming that every mate pair is from the same type as the first element. Affected #: 1 file diff -r 5c9cfd1dea8bfb85bcc1ee07035629a77ee3a051 -r 0282484e4c323e1baf79d05813ea8213239a8b4e lib/galaxy/tools/parameters/output.py --- a/lib/galaxy/tools/parameters/output.py +++ b/lib/galaxy/tools/parameters/output.py @@ -188,7 +188,14 @@ assert ref_name in value, "Required dependency '%s' not found in incoming values" % ref_name value = value.get( ref_name ) for attr_name in self.param_attribute: - value = getattr( value, attr_name ) + # if the value is a list from a repeat tag you can access the first element of the repeat with + # artifical 'first' attribute_name. For example: .. param_attribute="first.input_mate1.ext" + if isinstance(value, list) and attr_name == 'first': + value = value[0] + elif isinstance(value, dict): + value = value[ attr_name ] + else: + value = getattr( value, attr_name ) options = [ [ str( value ) ] ] for filter in self.filters: options = filter.filter_options( options, other_values ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: UI, Popupmenu: fix for IE 9; pack scripts
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5c9cfd1dea8b/ Changeset: 5c9cfd1dea8b User: carlfeberhard Date: 2013-10-17 22:05:29 Summary: UI, Popupmenu: fix for IE 9; pack scripts Affected #: 7 files diff -r f888228f48d89ade8b8f49a4d777c573174392d1 -r 5c9cfd1dea8bfb85bcc1ee07035629a77ee3a051 static/scripts/mvc/ui.js --- a/static/scripts/mvc/ui.js +++ b/static/scripts/mvc/ui.js @@ -213,226 +213,159 @@ }); // ============================================================================= -/** +/** * view for a popup menu */ -var PopupMenu = Backbone.View.extend( -{ - /* TODO: - add submenus - add hrefs - test various html keys - add make_popupmenus style - */ +var PopupMenu = Backbone.View.extend({ /** Cache the desired button element and options, set up the button click handler * NOTE: attaches this view as HTML/jQ data on the button for later use. */ - //TODO: include docs on special option keys (divider, checked, etc.) - initialize: function($button, options) - { + initialize: function( $button, options ){ // default settings - this.$button = $button || $('<div/>'); + this.$button = $button || $( '<div/>' ); this.options = options || []; // set up button click -> open menu behavior var menu = this; - this.$button.click(function(event) - { - menu._renderAndShow(event); + this.$button.click( function( event ){ + menu._renderAndShow( event ); return false; }); + }, - // attach this view as a data object on the button - for later access - //TODO:?? memleak? - this.$button.data('PopupMenu', this); + // render the menu, append to the page body at the click position, and set up the 'click-away' handlers, show + _renderAndShow: function( clickEvent ){ + this.render(); + this.$el.appendTo( 'body' ); + this.$el.css( this._getShownPosition( clickEvent )); + this._setUpCloseBehavior(); + this.$el.show(); }, // render the menu - // this menu doesn't attach itself to the DOM (see _renderAndShow) - render: function() - { - // link this popup - var menu = this; - - // render the menu body - this.$el.addClass('popmenu-wrapper') - .css( - { - position : 'absolute', - display : 'none' - }); - - // use template - this.$el.html(this.template(this.$button.attr('id'), this.options)); + // this menu doesn't attach itself to the DOM ( see _renderAndShow ) + render: function(){ + // render the menu body absolute and hidden, fill with template + this.$el.addClass( 'popmenu-wrapper' ).hide() + .css({ position : 'absolute' }) + .html( this.template( this.$button.attr( 'id' ), this.options )); // set up behavior on each link/anchor elem - if(this.options.length) - { - this.$el.find('li').each(function(i, li) - { - var $li = $(li), - $anchor = $li.children( 'a.popupmenu-option' ), - menuFunc = menu.options[i].func; + if( this.options.length ){ + var menu = this; + //precondition: there should be one option per li + this.$el.find( 'li' ).each( function( i, li ){ + var option = menu.options[i]; - // click event - if($anchor.length && menuFunc) - { - $anchor.click(function(event) - { - menuFunc(event, menu.options[i]); + // if the option has 'func', call that function when the anchor is clicked + if( option.func ){ + $( this ).children( 'a.popupmenu-option' ).click( function( event ){ + option.func.call( menu, event, option ); + // bubble up so that an option click will call the close behavior + //return false; }); } - - // cache the anchor as a jq obj within the options obj - menu.options[i].$li = $li; }); } return this; }, + template : function( id, options ){ + return [ + '<ul id="', id, '-menu" class="dropdown-menu">', this._templateOptions( options ), '</ul>' + ].join( '' ); + }, + + _templateOptions : function( options ){ + if( !options.length ){ + return '<li>(no options)</li>'; + } + return _.map( options, function( option ){ + if( option.divider ){ + return '<li class="divider"></li>'; + } else if( option.header ){ + return [ '<li class="head"><a href="javascript:void(0);">', option.html, '</a></li>' ].join( '' ); + } + var href = option.href || 'javascript:void(0);', + target = ( option.target )?( ' target="' + target + '"' ):( '' ), + check = ( option.checked )?( '<span class="fa-icon-ok"></span>' ):( '' ); + return [ + '<li><a class="popupmenu-option" href="', href, '"', target, '>', + check, option.html, + '</a></li>' + ].join( '' ); + }).join( '' ); + }, + // get the absolute position/offset for the menu - _getShownPosition : function( clickEvent ) - { - // get element width + _getShownPosition : function( clickEvent ){ + + // display menu horiz. centered on click... var menuWidth = this.$el.width(); - - // display menu horiz. centered on click... var x = clickEvent.pageX - menuWidth / 2 ; - // adjust to handle horiz. scroll and window dimensions (draw entirely on visible screen area) + // adjust to handle horiz. scroll and window dimensions ( draw entirely on visible screen area ) x = Math.min( x, $( document ).scrollLeft() + $( window ).width() - menuWidth - 5 ); x = Math.max( x, $( document ).scrollLeft() + 5 ); - - // return return { top: clickEvent.pageY, left: x }; }, - // render the menu, append to the page body at the click position, and set up the 'click-away' handlers, show - _renderAndShow: function(clickEvent) - { - this.render(); - this.$el.appendTo('body'); - this.$el.css( this._getShownPosition(clickEvent)); - this._setUpCloseBehavior(); - this.$el.show(); - }, - // bind an event handler to all available frames so that when anything is clicked // the menu is removed from the DOM and the event handler unbinds itself - _setUpCloseBehavior: function() - { + _setUpCloseBehavior: function(){ // function to close popup and unbind itself var menu = this; - var closePopupWhenClicked = function($elClicked) - { - $elClicked.bind("click.close_popup", function() - { + var closePopupWhenClicked = function( $elClicked ){ + $elClicked.one( "click.close_popup", function(){ menu.remove(); - $elClicked.unbind("click.close_popup"); }); }; // bind to current, parent, and sibling frames - closePopupWhenClicked($(window.document)); - closePopupWhenClicked($(window.top.document)); - _.each(window.top.frames, function(siblingFrame) - { - closePopupWhenClicked($(siblingFrame.document)); + closePopupWhenClicked( $( window.document )); + closePopupWhenClicked( $( window.top.document )); + _.each( window.top.frames, function( siblingFrame ){ + closePopupWhenClicked( $( siblingFrame.document )); }); }, // add a menu option/item at the given index - addItem: function(item, index) - { + addItem: function( item, index ){ // append to end if no index - index = (index >= 0) ? index : this.options.length; - this.options.splice(index, 0, item); + index = ( index >= 0 ) ? index : this.options.length; + this.options.splice( index, 0, item ); return this; }, // remove a menu option/item at the given index - removeItem: function(index) - { - if(index >=0) - this.options.splice(index, 1); + removeItem: function( index ){ + if( index >=0 ){ + this.options.splice( index, 1 ); + } return this; }, // search for a menu option by it's html - findIndexByHtml: function(html) - { - for(var i = 0; i < this.options.length; i++) - if(_.has(this.options[i], 'html') && (this.options[i].html === html)) + findIndexByHtml: function( html ){ + for( var i = 0; i < this.options.length; i++ ){ + if( _.has( this.options[i], 'html' ) && ( this.options[i].html === html )){ return i; + } + } return null; }, // search for a menu option by it's html - findItemByHtml: function(html) - { - return this.options[(this.findIndexByHtml(html))]; + findItemByHtml: function( html ){ + return this.options[( this.findIndexByHtml( html ))]; }, // string representation - toString: function() - { + toString: function(){ return 'PopupMenu'; - }, - - // template - template: function(id, options) - { - // initialize template - var tmpl = '<ul id="' + id + '-menu" class="dropdown-menu">'; - - // check item number - if (options.length > 0) - { - // add option - for (var i in options) - { - // get item - var item = options[i]; - - // check for divider - if (item.divider) - { - // add divider - tmpl += '<li class="divider"></li>'; - } else { - // identify header - if(item.header) - { - tmpl += '<li class="head"><a href="javascript:void(0);">' + item.html + '</a></li>'; - } else { - // add href - if (item.href) - { - tmpl += '<li><a href="' + item.href + '"'; - tmpl += 'target="' + item.target + '"'; - } else - tmpl += '<li><a href="javascript:void(0);"'; - - // add class - tmpl += 'class="popupmenu-option">' - - // add target - if (item.checked) - tmpl += '<span class="fa-icon-ok"></span>'; - - // add html - tmpl += item.html; - } - } - } - } else - tmpl += '<li>No Options.</li>'; - - // return - return tmpl + '</ul>'; } }); diff -r f888228f48d89ade8b8f49a4d777c573174392d1 -r 5c9cfd1dea8bfb85bcc1ee07035629a77ee3a051 static/scripts/packed/galaxy.frame.js --- a/static/scripts/packed/galaxy.frame.js +++ b/static/scripts/packed/galaxy.frame.js @@ -1,1 +1,1 @@ -define(["galaxy.master","libs/backbone/backbone-relational"],function(b){var a=Backbone.View.extend({el_main:"#everything",options:{frame:{cols:6,rows:3},rows:1000,cell:130,margin:5,scroll:5,top_min:40,frame_max:9},cols:0,top:0,top_max:0,frame_z:0,frame_counter:0,frame_counter_id:0,frame_list:[],frame_shadow:null,visible:false,active:false,button_active:null,button_load:null,initialize:function(d){var c=this;this.button_active=new b.GalaxyMasterIcon({icon:"fa-icon-th",tooltip:"Enable/Disable Scratchbook",on_click:function(f){c.event_panel_active(f)}});Galaxy.master.append(this.button_active);this.button_load=new b.GalaxyMasterIcon({icon:"fa-icon-eye-open",tooltip:"Show/Hide Scratchbook",on_click:function(f){c.event_panel_load(f)},with_number:true});Galaxy.master.append(this.button_load);if(d){this.options=_.defaults(d,this.options)}this.top=this.top_max=this.options.top_min;this.setElement(this.template());$(this.el).append(this.template_background());$(this.el).append(this.template_menu());$(this.el_main).append($(this.el));var e="#frame-shadow";$(this.el).append(this.template_shadow(e.substring(1)));this.frame_shadow={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_resize(this.frame_shadow,{width:0,height:0});this.frame_list[e]=this.frame_shadow;this.panel_refresh();var c=this;$(window).resize(function(){if(c.visible){c.panel_refresh()}});window.onbeforeunload=function(){if(c.frame_counter>0){return"You opened "+c.frame_counter+" frame(s) which will be lost."}}},event:{type:null,target:null,xy:null},events:{mousemove:"event_frame_mouse_move",mouseup:"event_frame_mouse_up",mouseleave:"event_frame_mouse_up",mousewheel:"event_panel_scroll",DOMMouseScroll:"event_panel_scroll","mousedown .frame":"event_frame_mouse_down","mousedown .frame-background":"event_panel_load","mousedown .frame-scroll-up":"event_panel_scroll_up","mousedown .frame-scroll-down":"event_panel_scroll_down","mousedown .f-close":"event_frame_close","mousedown .f-pin":"event_frame_lock"},event_frame_mouse_down:function(c){if(this.event.type!==null){return}if($(c.target).hasClass("f-header")||$(c.target).hasClass("f-title")){this.event.type="drag"}if($(c.target).hasClass("f-resize")){this.event.type="resize"}if(this.event.type===null){return}c.preventDefault();this.event.target=this.event_get_frame(c.target);if(this.event.target.grid_lock){this.event.type=null;return}this.event.xy={x:c.originalEvent.pageX,y:c.originalEvent.pageY};this.frame_drag_start(this.event.target)},event_frame_mouse_move:function(i){if(this.event.type!="drag"&&this.event.type!="resize"){return}var g={x:i.originalEvent.pageX,y:i.originalEvent.pageY};var d={x:g.x-this.event.xy.x,y:g.y-this.event.xy.y};this.event.xy=g;var h=this.frame_screen(this.event.target);if(this.event.type=="resize"){h.width+=d.x;h.height+=d.y;var f=this.options.cell-this.options.margin-1;h.width=Math.max(h.width,f);h.height=Math.max(h.height,f);this.frame_resize(this.event.target,h);h.width=this.to_grid_coord("width",h.width)+1;h.height=this.to_grid_coord("height",h.height)+1;h.width=this.to_pixel_coord("width",h.width);h.height=this.to_pixel_coord("height",h.height);this.frame_resize(this.frame_shadow,h);this.frame_insert(this.frame_shadow,{top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)})}if(this.event.type=="drag"){h.left+=d.x;h.top+=d.y;this.frame_offset(this.event.target,h);var c={top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)};if(c.left!==0){c.left++}this.frame_insert(this.frame_shadow,c)}},event_frame_mouse_up:function(c){if(this.event.type!="drag"&&this.event.type!="resize"){return}this.frame_drag_stop(this.event.target);this.event.type=null},event_frame_close:function(d){if(this.event.type!==null){return}d.preventDefault();var f=this.event_get_frame(d.target);var c=this;$(f.id).fadeOut("fast",function(){$(f.id).remove();delete c.frame_list[f.id];c.frame_counter--;c.panel_refresh(true);c.panel_animation_complete();if(c.visible&&c.frame_counter==0){c.panel_show_hide()}})},event_frame_lock:function(c){if(this.event.type!==null){return}c.preventDefault();var d=this.event_get_frame(c.target);if(d.grid_lock){d.grid_lock=false;$(d.id).find(".f-pin").removeClass("toggle");$(d.id).find(".f-header").removeClass("f-not-allowed");$(d.id).find(".f-title").removeClass("f-not-allowed");$(d.id).find(".f-resize").show();$(d.id).find(".f-close").show()}else{d.grid_lock=true;$(d.id).find(".f-pin").addClass("toggle");$(d.id).find(".f-header").addClass("f-not-allowed");$(d.id).find(".f-title").addClass("f-not-allowed");$(d.id).find(".f-resize").hide();$(d.id).find(".f-close").hide()}},event_panel_load:function(c){if(this.event.type!==null){return}this.panel_show_hide()},event_panel_active:function(c){if(this.event.type!==null){return}this.panel_active_disable()},event_panel_scroll:function(c){if(this.event.type!==null||!this.visible){return}c.preventDefault();var d=c.originalEvent.detail?c.originalEvent.detail:c.originalEvent.wheelDelta/-3;this.panel_scroll(d)},event_panel_scroll_up:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(-this.options.scroll)},event_panel_scroll_down:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(this.options.scroll)},event_get_frame:function(c){return this.frame_list["#"+$(c).closest(".frame").attr("id")]},frame_drag_start:function(d){this.frame_focus(d,true);var c=this.frame_screen(d);this.frame_resize(this.frame_shadow,c);this.frame_grid(this.frame_shadow,d.grid_location);d.grid_location=null;$(this.frame_shadow.id).show();$(".f-cover").show()},frame_drag_stop:function(d){this.frame_focus(d,false);var c=this.frame_screen(this.frame_shadow);this.frame_resize(d,c);this.frame_grid(d,this.frame_shadow.grid_location,true);this.frame_shadow.grid_location=null;$(this.frame_shadow.id).hide();$(".f-cover").hide();this.panel_animation_complete()},to_grid_coord:function(e,d){var c=(e=="width"||e=="height")?1:-1;if(e=="top"){d-=this.top}return parseInt((d+c*this.options.margin)/this.options.cell,10)},to_pixel_coord:function(e,f){var c=(e=="width"||e=="height")?1:-1;var d=(f*this.options.cell)-c*this.options.margin;if(e=="top"){d+=this.top}return d},to_grid:function(c){return{top:this.to_grid_coord("top",c.top),left:this.to_grid_coord("left",c.left),width:this.to_grid_coord("width",c.width),height:this.to_grid_coord("height",c.height)}},to_pixel:function(c){return{top:this.to_pixel_coord("top",c.top),left:this.to_pixel_coord("left",c.left),width:this.to_pixel_coord("width",c.width),height:this.to_pixel_coord("height",c.height)}},is_collision:function(e){function c(h,g){return !(h.left>g.left+g.width-1||h.left+h.width-1<g.left||h.top>g.top+g.height-1||h.top+h.height-1<g.top)}for(var d in this.frame_list){var f=this.frame_list[d];if(f.grid_location===null){continue}if(c(e,f.grid_location)){return true}}return false},location_rank:function(c){return(c.top*this.cols)+c.left},menu_refresh:function(){this.button_load.number(this.frame_counter);if(this.frame_counter==0){this.button_load.hide()}else{this.button_load.show()}if(this.top==this.options.top_min){$(".frame-scroll-up").hide()}else{$(".frame-scroll-up").show()}if(this.top==this.top_max){$(".frame-scroll-down").hide()}else{$(".frame-scroll-down").show()}},panel_animation_complete:function(){var c=this;$(".frame").promise().done(function(){c.panel_scroll(0,true)})},panel_refresh:function(c){this.cols=parseInt($(window).width()/this.options.cell,10)+1;this.frame_insert(null,null,c)},panel_scroll:function(h,c){var e=this.top-this.options.scroll*h;e=Math.max(e,this.top_max);e=Math.min(e,this.options.top_min);if(this.top!=e){for(var d in this.frame_list){var g=this.frame_list[d];if(g.grid_location!==null){var f={top:g.screen_location.top-(this.top-e),left:g.screen_location.left};this.frame_offset(g,f,c)}}this.top=e}this.menu_refresh()},panel_show_hide:function(){if(this.visible){this.visible=false;$(".frame").fadeOut("fast");this.button_load.icon("fa-icon-eye-close");this.button_load.untoggle();$(".frame-background").hide();$(".frame-menu").hide()}else{this.visible=true;$(".frame").fadeIn("fast");this.button_load.icon("fa-icon-eye-open");this.button_load.toggle();$(this.frame_shadow.id).hide();$(".frame-background").show();this.panel_refresh()}},panel_active_disable:function(){if(this.active){this.active=false;this.button_active.untoggle();if(this.visible){this.panel_show_hide()}}else{this.active=true;this.button_active.toggle()}},frame_new:function(d){if(!this.active){if(d.location=="center"){var c=$(window.parent.document).find("iframe#galaxy_main");c.attr("src",d.content)}else{window.location=d.content}return}if(this.frame_counter>=this.options.frame_max){alert("You have reached the maximum number of allowed frames ("+this.options.frame_max+").");return}var e="#frame-"+(this.frame_counter_id++);if($(e).length!==0){alert("This frame already exists. This page might contain multiple frame managers.");return}this.top=this.options.top_min;$(this.el).append(this.template_frame(e.substring(1),d.title,d.type,d.content));var f={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};d.width=this.to_pixel_coord("width",this.options.frame.cols);d.height=this.to_pixel_coord("height",this.options.frame.rows);this.frame_z=parseInt($(f.id).css("z-index"));this.frame_list[e]=f;this.frame_counter++;this.frame_resize(f,{width:d.width,height:d.height});this.frame_insert(f,{top:0,left:0},true);if(!this.visible){this.panel_show_hide()}},frame_insert:function(j,c,e){var d=[];if(j){j.grid_location=null;d.push([j,this.location_rank(c)])}var g=null;for(g in this.frame_list){var h=this.frame_list[g];if(h.grid_location!==null&&!h.grid_lock){h.grid_location=null;d.push([h,h.grid_rank])}}d.sort(function(k,f){var m=k[1];var l=f[1];return m<l?-1:(m>l?1:0)});for(g=0;g<d.length;g++){this.frame_place(d[g][0],e)}this.top_max=0;for(var g in this.frame_list){var j=this.frame_list[g];if(j.grid_location!==null){this.top_max=Math.max(this.top_max,j.grid_location.top+j.grid_location.height)}}this.top_max=$(window).height()-this.top_max*this.options.cell-2*this.options.margin;this.top_max=Math.min(this.top_max,this.options.top_min);this.menu_refresh()},frame_place:function(k,d){k.grid_location=null;var h=this.to_grid(this.frame_screen(k));var c=false;for(var f=0;f<this.options.rows;f++){for(var e=0;e<Math.max(1,this.cols-h.width);e++){h.top=f;h.left=e;if(!this.is_collision(h)){c=true;break}}if(c){break}}if(c){this.frame_grid(k,h,d)}else{console.log("Grid dimensions exceeded.")}},frame_focus:function(e,c){var d=this.frame_z+(c?1:0);$(e.id).css("z-index",d)},frame_offset:function(f,e,d){f.screen_location.left=e.left;f.screen_location.top=e.top;if(d){this.frame_focus(f,true);var c=this;$(f.id).animate({top:e.top,left:e.left},"fast",function(){c.frame_focus(f,false)})}else{$(f.id).css({top:e.top,left:e.left})}},frame_resize:function(d,c){$(d.id).css({width:c.width,height:c.height});d.screen_location.width=c.width;d.screen_location.height=c.height},frame_grid:function(e,c,d){e.grid_location=c;this.frame_offset(e,this.to_pixel(c),d);e.grid_rank=this.location_rank(c)},frame_screen:function(d){var c=d.screen_location;return{top:c.top,left:c.left,width:c.width,height:c.height}},template:function(){return'<div class="galaxy-frame"></div>'},template_frame:function(f,e,c,d){if(!e){e=""}if(c=="url"){d='<iframe scrolling="auto" class="f-iframe" src="'+d+'"></iframe>'}return'<div id="'+f+'" class="frame corner"><div class="f-header corner"><span class="f-title">'+e+'</span><span class="f-icon f-pin fa-icon-pushpin"></span><span class="f-icon f-close fa-icon-trash"></span></div><div class="f-content">'+d+'<div class="f-cover"></div></div><span class="f-resize f-icon corner fa-icon-resize-full"></span></div>'},template_shadow:function(c){return'<div id="'+c+'" class="frame-shadow corner"></div>'},template_background:function(){return'<div class="frame-background"></div>'},template_menu:function(){return'<div class="frame-scroll-up frame-menu fa-icon-chevron-up fa-icon-2x"></div><div class="frame-scroll-down frame-menu fa-icon-chevron-down fa-icon-2x"></div>'}});return{GalaxyFrameManager:a}}); \ No newline at end of file +define(["galaxy.master","libs/backbone/backbone-relational"],function(b){var a=Backbone.View.extend({el_main:"#everything",options:{frame:{cols:6,rows:3},rows:1000,cell:130,margin:5,scroll:5,top_min:40,frame_max:9},cols:0,top:0,top_max:0,frame_z:0,frame_counter:0,frame_counter_id:0,frame_list:[],frame_shadow:null,visible:false,active:false,button_active:null,button_load:null,initialize:function(d){var c=this;this.button_active=new b.GalaxyMasterIcon({icon:"fa-icon-th",tooltip:"Enable/Disable Scratchbook",on_click:function(f){c.event_panel_active(f)},on_unload:function(){if(c.frame_counter>0){return"You opened "+c.frame_counter+" frame(s) which will be lost."}}});Galaxy.master.append(this.button_active);this.button_load=new b.GalaxyMasterIcon({icon:"fa-icon-eye-open",tooltip:"Show/Hide Scratchbook",on_click:function(f){c.event_panel_load(f)},with_number:true});Galaxy.master.append(this.button_load);if(d){this.options=_.defaults(d,this.options)}this.top=this.top_max=this.options.top_min;this.setElement(this.template());$(this.el).append(this.template_background());$(this.el).append(this.template_menu());$(this.el_main).append($(this.el));var e="#frame-shadow";$(this.el).append(this.template_shadow(e.substring(1)));this.frame_shadow={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_resize(this.frame_shadow,{width:0,height:0});this.frame_list[e]=this.frame_shadow;this.panel_refresh();var c=this;$(window).resize(function(){if(c.visible){c.panel_refresh()}})},event:{type:null,target:null,xy:null},events:{mousemove:"event_frame_mouse_move",mouseup:"event_frame_mouse_up",mouseleave:"event_frame_mouse_up",mousewheel:"event_panel_scroll",DOMMouseScroll:"event_panel_scroll","mousedown .frame":"event_frame_mouse_down","mousedown .frame-background":"event_panel_load","mousedown .frame-scroll-up":"event_panel_scroll_up","mousedown .frame-scroll-down":"event_panel_scroll_down","mousedown .f-close":"event_frame_close","mousedown .f-pin":"event_frame_lock"},event_frame_mouse_down:function(c){if(this.event.type!==null){return}if($(c.target).hasClass("f-header")||$(c.target).hasClass("f-title")){this.event.type="drag"}if($(c.target).hasClass("f-resize")){this.event.type="resize"}if(this.event.type===null){return}c.preventDefault();this.event.target=this.event_get_frame(c.target);if(this.event.target.grid_lock){this.event.type=null;return}this.event.xy={x:c.originalEvent.pageX,y:c.originalEvent.pageY};this.frame_drag_start(this.event.target)},event_frame_mouse_move:function(i){if(this.event.type!="drag"&&this.event.type!="resize"){return}var g={x:i.originalEvent.pageX,y:i.originalEvent.pageY};var d={x:g.x-this.event.xy.x,y:g.y-this.event.xy.y};this.event.xy=g;var h=this.frame_screen(this.event.target);if(this.event.type=="resize"){h.width+=d.x;h.height+=d.y;var f=this.options.cell-this.options.margin-1;h.width=Math.max(h.width,f);h.height=Math.max(h.height,f);this.frame_resize(this.event.target,h);h.width=this.to_grid_coord("width",h.width)+1;h.height=this.to_grid_coord("height",h.height)+1;h.width=this.to_pixel_coord("width",h.width);h.height=this.to_pixel_coord("height",h.height);this.frame_resize(this.frame_shadow,h);this.frame_insert(this.frame_shadow,{top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)})}if(this.event.type=="drag"){h.left+=d.x;h.top+=d.y;this.frame_offset(this.event.target,h);var c={top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)};if(c.left!==0){c.left++}this.frame_insert(this.frame_shadow,c)}},event_frame_mouse_up:function(c){if(this.event.type!="drag"&&this.event.type!="resize"){return}this.frame_drag_stop(this.event.target);this.event.type=null},event_frame_close:function(d){if(this.event.type!==null){return}d.preventDefault();var f=this.event_get_frame(d.target);var c=this;$(f.id).fadeOut("fast",function(){$(f.id).remove();delete c.frame_list[f.id];c.frame_counter--;c.panel_refresh(true);c.panel_animation_complete();if(c.visible&&c.frame_counter==0){c.panel_show_hide()}})},event_frame_lock:function(c){if(this.event.type!==null){return}c.preventDefault();var d=this.event_get_frame(c.target);if(d.grid_lock){d.grid_lock=false;$(d.id).find(".f-pin").removeClass("toggle");$(d.id).find(".f-header").removeClass("f-not-allowed");$(d.id).find(".f-title").removeClass("f-not-allowed");$(d.id).find(".f-resize").show();$(d.id).find(".f-close").show()}else{d.grid_lock=true;$(d.id).find(".f-pin").addClass("toggle");$(d.id).find(".f-header").addClass("f-not-allowed");$(d.id).find(".f-title").addClass("f-not-allowed");$(d.id).find(".f-resize").hide();$(d.id).find(".f-close").hide()}},event_panel_load:function(c){if(this.event.type!==null){return}this.panel_show_hide()},event_panel_active:function(c){if(this.event.type!==null){return}this.panel_active_disable()},event_panel_scroll:function(c){if(this.event.type!==null||!this.visible){return}c.preventDefault();var d=c.originalEvent.detail?c.originalEvent.detail:c.originalEvent.wheelDelta/-3;this.panel_scroll(d)},event_panel_scroll_up:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(-this.options.scroll)},event_panel_scroll_down:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(this.options.scroll)},event_get_frame:function(c){return this.frame_list["#"+$(c).closest(".frame").attr("id")]},frame_drag_start:function(d){this.frame_focus(d,true);var c=this.frame_screen(d);this.frame_resize(this.frame_shadow,c);this.frame_grid(this.frame_shadow,d.grid_location);d.grid_location=null;$(this.frame_shadow.id).show();$(".f-cover").show()},frame_drag_stop:function(d){this.frame_focus(d,false);var c=this.frame_screen(this.frame_shadow);this.frame_resize(d,c);this.frame_grid(d,this.frame_shadow.grid_location,true);this.frame_shadow.grid_location=null;$(this.frame_shadow.id).hide();$(".f-cover").hide();this.panel_animation_complete()},to_grid_coord:function(e,d){var c=(e=="width"||e=="height")?1:-1;if(e=="top"){d-=this.top}return parseInt((d+c*this.options.margin)/this.options.cell,10)},to_pixel_coord:function(e,f){var c=(e=="width"||e=="height")?1:-1;var d=(f*this.options.cell)-c*this.options.margin;if(e=="top"){d+=this.top}return d},to_grid:function(c){return{top:this.to_grid_coord("top",c.top),left:this.to_grid_coord("left",c.left),width:this.to_grid_coord("width",c.width),height:this.to_grid_coord("height",c.height)}},to_pixel:function(c){return{top:this.to_pixel_coord("top",c.top),left:this.to_pixel_coord("left",c.left),width:this.to_pixel_coord("width",c.width),height:this.to_pixel_coord("height",c.height)}},is_collision:function(e){function c(h,g){return !(h.left>g.left+g.width-1||h.left+h.width-1<g.left||h.top>g.top+g.height-1||h.top+h.height-1<g.top)}for(var d in this.frame_list){var f=this.frame_list[d];if(f.grid_location===null){continue}if(c(e,f.grid_location)){return true}}return false},location_rank:function(c){return(c.top*this.cols)+c.left},menu_refresh:function(){this.button_load.number(this.frame_counter);if(this.frame_counter==0){this.button_load.hide()}else{this.button_load.show()}if(this.top==this.options.top_min){$(".frame-scroll-up").hide()}else{$(".frame-scroll-up").show()}if(this.top==this.top_max){$(".frame-scroll-down").hide()}else{$(".frame-scroll-down").show()}},panel_animation_complete:function(){var c=this;$(".frame").promise().done(function(){c.panel_scroll(0,true)})},panel_refresh:function(c){this.cols=parseInt($(window).width()/this.options.cell,10)+1;this.frame_insert(null,null,c)},panel_scroll:function(h,c){var e=this.top-this.options.scroll*h;e=Math.max(e,this.top_max);e=Math.min(e,this.options.top_min);if(this.top!=e){for(var d in this.frame_list){var g=this.frame_list[d];if(g.grid_location!==null){var f={top:g.screen_location.top-(this.top-e),left:g.screen_location.left};this.frame_offset(g,f,c)}}this.top=e}this.menu_refresh()},panel_show_hide:function(){if(this.visible){this.visible=false;$(".frame").fadeOut("fast");this.button_load.icon("fa-icon-eye-close");this.button_load.untoggle();$(".frame-background").hide();$(".frame-menu").hide()}else{this.visible=true;$(".frame").fadeIn("fast");this.button_load.icon("fa-icon-eye-open");this.button_load.toggle();$(this.frame_shadow.id).hide();$(".frame-background").show();this.panel_refresh()}},panel_active_disable:function(){if(this.active){this.active=false;this.button_active.untoggle();if(this.visible){this.panel_show_hide()}}else{this.active=true;this.button_active.toggle()}},frame_new:function(d){if(!this.active){if(d.location=="center"){var c=$(window.parent.document).find("iframe#galaxy_main");c.attr("src",d.content)}else{window.location=d.content}return}if(this.frame_counter>=this.options.frame_max){alert("You have reached the maximum number of allowed frames ("+this.options.frame_max+").");return}var e="#frame-"+(this.frame_counter_id++);if($(e).length!==0){alert("This frame already exists. This page might contain multiple frame managers.");return}this.top=this.options.top_min;$(this.el).append(this.template_frame(e.substring(1),d.title,d.type,d.content));var f={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};d.width=this.to_pixel_coord("width",this.options.frame.cols);d.height=this.to_pixel_coord("height",this.options.frame.rows);this.frame_z=parseInt($(f.id).css("z-index"));this.frame_list[e]=f;this.frame_counter++;this.frame_resize(f,{width:d.width,height:d.height});this.frame_insert(f,{top:0,left:0},true);if(!this.visible){this.panel_show_hide()}},frame_insert:function(j,c,e){var d=[];if(j){j.grid_location=null;d.push([j,this.location_rank(c)])}var g=null;for(g in this.frame_list){var h=this.frame_list[g];if(h.grid_location!==null&&!h.grid_lock){h.grid_location=null;d.push([h,h.grid_rank])}}d.sort(function(k,f){var m=k[1];var l=f[1];return m<l?-1:(m>l?1:0)});for(g=0;g<d.length;g++){this.frame_place(d[g][0],e)}this.top_max=0;for(var g in this.frame_list){var j=this.frame_list[g];if(j.grid_location!==null){this.top_max=Math.max(this.top_max,j.grid_location.top+j.grid_location.height)}}this.top_max=$(window).height()-this.top_max*this.options.cell-2*this.options.margin;this.top_max=Math.min(this.top_max,this.options.top_min);this.menu_refresh()},frame_place:function(k,d){k.grid_location=null;var h=this.to_grid(this.frame_screen(k));var c=false;for(var f=0;f<this.options.rows;f++){for(var e=0;e<Math.max(1,this.cols-h.width);e++){h.top=f;h.left=e;if(!this.is_collision(h)){c=true;break}}if(c){break}}if(c){this.frame_grid(k,h,d)}else{console.log("Grid dimensions exceeded.")}},frame_focus:function(e,c){var d=this.frame_z+(c?1:0);$(e.id).css("z-index",d)},frame_offset:function(f,e,d){f.screen_location.left=e.left;f.screen_location.top=e.top;if(d){this.frame_focus(f,true);var c=this;$(f.id).animate({top:e.top,left:e.left},"fast",function(){c.frame_focus(f,false)})}else{$(f.id).css({top:e.top,left:e.left})}},frame_resize:function(d,c){$(d.id).css({width:c.width,height:c.height});d.screen_location.width=c.width;d.screen_location.height=c.height},frame_grid:function(e,c,d){e.grid_location=c;this.frame_offset(e,this.to_pixel(c),d);e.grid_rank=this.location_rank(c)},frame_screen:function(d){var c=d.screen_location;return{top:c.top,left:c.left,width:c.width,height:c.height}},template:function(){return'<div class="galaxy-frame"></div>'},template_frame:function(f,e,c,d){if(!e){e=""}if(c=="url"){d='<iframe scrolling="auto" class="f-iframe" src="'+d+'"></iframe>'}return'<div id="'+f+'" class="frame corner"><div class="f-header corner"><span class="f-title">'+e+'</span><span class="f-icon f-pin fa-icon-pushpin"></span><span class="f-icon f-close fa-icon-trash"></span></div><div class="f-content">'+d+'<div class="f-cover"></div></div><span class="f-resize f-icon corner fa-icon-resize-full"></span></div>'},template_shadow:function(c){return'<div id="'+c+'" class="frame-shadow corner"></div>'},template_background:function(){return'<div class="frame-background"></div>'},template_menu:function(){return'<div class="frame-scroll-up frame-menu fa-icon-chevron-up fa-icon-2x"></div><div class="frame-scroll-down frame-menu fa-icon-chevron-down fa-icon-2x"></div>'}});return{GalaxyFrameManager:a}}); \ No newline at end of file diff -r f888228f48d89ade8b8f49a4d777c573174392d1 -r 5c9cfd1dea8bfb85bcc1ee07035629a77ee3a051 static/scripts/packed/galaxy.master.js --- a/static/scripts/packed/galaxy.master.js +++ b/static/scripts/packed/galaxy.master.js @@ -1,1 +1,1 @@ -define(["libs/backbone/backbone-relational"],function(){var a=Backbone.View.extend({el_master:"#masthead",initialize:function(c){this.setElement($(this.template()));$(this.el_master).append($(this.el))},events:{mousedown:function(c){c.preventDefault()}},append:function(c){$(this.el).append($(c.el))},prepend:function(c){$(this.el).prepend($(c.el))},template:function(){return'<div class="iconbar"></div>'}});var b=Backbone.View.extend({options:{id:"galaxy-icon",icon:"fa-icon-cog",tooltip:"galaxy-icon",with_number:false,on_click:function(){alert("clicked")},visible:true},initialize:function(d){if(d){this.options=_.defaults(d,this.options)}this.setElement($(this.template(this.options)));var c=this;$(this.el).find(".icon").tooltip({title:this.options.tooltip}).on("click",c.options.on_click);if(!this.options.visible){this.hide()}},show:function(){$(this.el).css({visibility:"visible"})},hide:function(){$(this.el).css({visibility:"hidden"})},icon:function(c){$(this.el).find(".icon").removeClass(this.options.icon).addClass(c);this.options.icon=c},toggle:function(){$(this.el).addClass("toggle")},untoggle:function(){$(this.el).removeClass("toggle")},number:function(c){$(this.el).find(".number").text(c)},template:function(d){var c="<div id="+d.id+' class="symbol"><div class="icon fa-icon-2x '+d.icon+'"></div>';if(d.with_number){c+='<div class="number"></div>'}c+="</div>";return c}});return{GalaxyMaster:a,GalaxyMasterIcon:b}}); \ No newline at end of file +define(["libs/backbone/backbone-relational"],function(){var a=Backbone.View.extend({el_master:"#masthead",list:[],initialize:function(d){this.setElement($(this.template()));$(this.el_master).append($(this.el));var c=this;window.onbeforeunload=function(){var f="";for(key in c.list){if(c.list[key].options.on_unload){var e=c.list[key].options.on_unload();if(e){f+=e+" "}}}if(f!=""){return f}}},events:{mousedown:function(c){c.preventDefault()}},append:function(c){$(this.el).append($(c.el));this.list.push(c)},prepend:function(c){$(this.el).prepend($(c.el));this.list.push(c)},template:function(){return'<div class="iconbar"></div>'}});var b=Backbone.View.extend({options:{id:"galaxy-icon",icon:"fa-icon-cog",tooltip:"galaxy-icon",with_number:false,on_click:function(){alert("clicked")},on_unload:null,visible:true},initialize:function(d){if(d){this.options=_.defaults(d,this.options)}this.setElement($(this.template(this.options)));var c=this;$(this.el).find(".icon").tooltip({title:this.options.tooltip}).on("click",c.options.on_click);if(!this.options.visible){this.hide()}},show:function(){$(this.el).css({visibility:"visible"})},hide:function(){$(this.el).css({visibility:"hidden"})},icon:function(c){$(this.el).find(".icon").removeClass(this.options.icon).addClass(c);this.options.icon=c},toggle:function(){$(this.el).addClass("toggle")},untoggle:function(){$(this.el).removeClass("toggle")},number:function(c){$(this.el).find(".number").text(c)},template:function(d){var c="<div id="+d.id+' class="symbol"><div class="icon fa-icon-2x '+d.icon+'"></div>';if(d.with_number){c+='<div class="number"></div>'}c+="</div>";return c}});return{GalaxyMaster:a,GalaxyMasterIcon:b}}); \ No newline at end of file diff -r f888228f48d89ade8b8f49a4d777c573174392d1 -r 5c9cfd1dea8bfb85bcc1ee07035629a77ee3a051 static/scripts/packed/galaxy.modal.js --- a/static/scripts/packed/galaxy.modal.js +++ b/static/scripts/packed/galaxy.modal.js @@ -1,1 +1,1 @@ -define(["libs/backbone/backbone-relational"],function(){var a=Backbone.View.extend({elMain:"#everything",optionsDefault:{title:"galaxy-modal",body:"",backdrop:true},options:{},initialize:function(b){if(b){this.create(b)}},show:function(b){this.initialize(b);var c=$(document).height()/2;this.$body.css("max-height",c);if(this.options.height){this.$body.css("height",Math.min(this.options.height,c))}if(this.options.height){this.$body.css("overflow","hidden")}if(this.visible){this.$el.show()}else{this.$el.fadeIn("fast")}this.visible=true},hide:function(){this.$el.fadeOut("fast");this.visible=false},create:function(c){this.options=_.defaults(c,this.optionsDefault);if(this.options.body=="progress"){this.options.body=$('<div class="progress progress-striped active"><div class="progress-bar progress-bar-info" style="width:100%"></div></div>')}if(this.$el){this.$el.remove()}this.setElement(this.template(this.options.title));this.$body=(this.$el).find(".modal-body");this.$footer=(this.$el).find(".modal-footer");this.$buttons=(this.$el).find(".buttons");this.$backdrop=(this.$el).find(".modal-backdrop");this.$body.html(this.options.body);this.$body.css("min-width",this.$body.width());if(!this.options.backdrop){this.$backdrop.removeClass("in")}if(this.options.buttons){var b=this;$.each(this.options.buttons,function(d,e){b.$buttons.append($('<button id="'+String(d).toLowerCase()+'"></button>').text(d).click(e)).append(" ")})}else{this.$footer.hide()}$(this.elMain).append($(this.el))},enableButton:function(b){this.$buttons.find("#"+String(b).toLowerCase()).prop("disabled",false)},disableButton:function(b){this.$buttons.find("#"+String(b).toLowerCase()).prop("disabled",true)},scrollTop:function(){return this.$body.scrollTop()},template:function(b){return'<div class="modal"><div class="modal-backdrop fade in" style="z-index: -1;"></div><div class="modal-dialog"><div class="modal-content"><div class="modal-header"><button type="button" class="close" style="display: none;">×</button><h4 class="title">'+b+'</h4></div><div class="modal-body"></div><div class="modal-footer"><div class="buttons" style="float: right;"></div></div></div</div></div>'}});return{GalaxyModal:a}}); \ No newline at end of file +define(["libs/backbone/backbone-relational"],function(){var a=Backbone.View.extend({elMain:"#everything",optionsDefault:{title:"galaxy-modal",body:"",backdrop:true,height:null,width:null},options:{},initialize:function(b){if(b){this.create(b)}},show:function(b){this.initialize(b);if(this.options.height){this.$body.css("height",this.options.height);this.$body.css("overflow","hidden")}else{this.$body.css("max-height",$(window).height()/2)}if(this.options.width){this.$dialog.css("width",this.options.width)}if(this.visible){this.$el.show()}else{this.$el.fadeIn("fast")}this.visible=true},hide:function(){this.$el.fadeOut("fast");this.visible=false},create:function(c){this.options=_.defaults(c,this.optionsDefault);if(this.options.body=="progress"){this.options.body=$('<div class="progress progress-striped active"><div class="progress-bar progress-bar-info" style="width:100%"></div></div>')}if(this.$el){this.$el.remove()}this.setElement(this.template(this.options.title));this.$dialog=(this.$el).find(".modal-dialog");this.$body=(this.$el).find(".modal-body");this.$footer=(this.$el).find(".modal-footer");this.$buttons=(this.$el).find(".buttons");this.$backdrop=(this.$el).find(".modal-backdrop");this.$body.html(this.options.body);if(!this.options.backdrop){this.$backdrop.removeClass("in")}if(this.options.buttons){var b=this;$.each(this.options.buttons,function(d,e){b.$buttons.append($('<button id="'+String(d).toLowerCase()+'"></button>').text(d).click(e)).append(" ")})}else{this.$footer.hide()}$(this.elMain).append($(this.el))},enableButton:function(b){this.$buttons.find("#"+String(b).toLowerCase()).prop("disabled",false)},disableButton:function(b){this.$buttons.find("#"+String(b).toLowerCase()).prop("disabled",true)},scrollTop:function(){return this.$body.scrollTop()},template:function(b){return'<div class="modal"><div class="modal-backdrop fade in" style="z-index: -1;"></div><div class="modal-dialog"><div class="modal-content"><div class="modal-header"><button type="button" class="close" style="display: none;">×</button><h4 class="title">'+b+'</h4></div><div class="modal-body"></div><div class="modal-footer"><div class="buttons" style="float: right;"></div></div></div</div></div>'}});return{GalaxyModal:a}}); \ No newline at end of file diff -r f888228f48d89ade8b8f49a4d777c573174392d1 -r 5c9cfd1dea8bfb85bcc1ee07035629a77ee3a051 static/scripts/packed/galaxy.upload.js --- a/static/scripts/packed/galaxy.upload.js +++ b/static/scripts/packed/galaxy.upload.js @@ -1,1 +1,1 @@ -define(["galaxy.modal","galaxy.master","utils/galaxy.utils","utils/galaxy.uploadbox","libs/backbone/backbone-relational"],function(b,d,c){var a=Backbone.View.extend({modal:null,button_show:null,uploadbox:null,select_extension:{auto:"Auto-detect"},state:{init:"fa-icon-trash",queued:"fa-icon-spinner fa-icon-spin",running:"__running__",success:"fa-icon-ok",error:"fa-icon-warning-sign"},counter:{announce:0,success:0,error:0,running:0,reset:function(){this.announce=this.success=this.error=this.running=0}},initialize:function(){if(!Galaxy.currHistoryPanel){var e=this;window.setTimeout(function(){e.initialize()},500);return}var e=this;this.button_show=new d.GalaxyMasterIcon({icon:"fa-icon-upload",tooltip:"Upload Files",on_click:function(f){e.event_show(f)},with_number:true});Galaxy.master.prepend(this.button_show);var e=this;c.jsonFromUrl(galaxy_config.root+"api/datatypes",function(g){for(key in g){var f=g[key];e.select_extension[f]=f}})},event_dragover:function(f){},event_dragleave:function(f){},event_announce:function(f,g,i){var j="#upload-"+f;$(this.el).find("tbody:last").append(this.template_row(j,this.select_extension));var h=this.get_upload_item(f);h.fadeIn();h.find("#title").html(g.name);h.find("#size").html(this.size_to_string(g.size));var e=this;h.find("#symbol").on("click",function(){e.event_remove(f)});this.event_progress(f,g,0);this.counter.announce++;this.update_screen()},event_initialize:function(g,h,k){this.button_show.number(this.counter.announce);var i=this.get_upload_item(g);var l=i.find("#symbol");l.addClass(this.state.running);var e=Galaxy.currHistoryPanel.model.get("id");var f=i.find("#extension").val();var j=i.find("#space_to_tabs").is(":checked");this.uploadbox.configure({url:galaxy_config.root+"api/tools/",paramname:"files_0|file_data"});tool_input={};tool_input.dbkey="?";tool_input.file_type=f;tool_input["files_0|NAME"]=h.name;tool_input["files_0|type"]="upload_dataset";tool_input.space_to_tabs=j;data={};data.history_id=e;data.tool_id="upload1";data.inputs=JSON.stringify(tool_input);return data},event_progress:function(f,g,i){var h=this.get_upload_item(f);var e=parseInt(i);h.find(".progress-bar").css({width:e+"%"});if(e!=100){h.find("#percentage").html(e+"%")}else{h.find("#percentage").html("Adding to history...")}},event_success:function(e,f,h){this.event_progress(e,f,100);this.button_show.number("");this.counter.announce--;this.counter.success++;this.update_screen();var g=this.get_upload_item(e);g.addClass("success");g.find("#percentage").html("100%");var i=g.find("#symbol");i.removeClass(this.state.running);i.removeClass(this.state.queued);i.addClass(this.state.success);Galaxy.currHistoryPanel.refresh()},event_error:function(e,f,h){this.event_progress(e,f,0);this.button_show.number("");this.counter.announce--;this.counter.error++;this.update_screen();var g=this.get_upload_item(e);g.addClass("danger");g.find(".progress").remove();g.find("#info").html("<strong>Failed: </strong>"+h).show();var i=g.find("#symbol");i.removeClass(this.state.running);i.removeClass(this.state.queued);i.addClass(this.state.error)},event_upload:function(){if(this.counter.announce==0||this.counter.running>0){return}var f=$(this.el).find(".upload-item");var e=this;f.each(function(){var g=$(this).find("#symbol");if(g.hasClass(e.state.init)){g.removeClass(e.state.init);g.addClass(e.state.queued);$(this).find("#extension").attr("disabled",true);$(this).find("#space_to_tabs").attr("disabled",true)}});this.counter.running=this.counter.announce;this.update_screen();this.uploadbox.upload()},event_pause:function(){if(this.counter.running==0){return}this.uploadbox.pause();$("#upload-info").html("Queueing will pause after completing the current file...")},event_complete:function(){this.counter.running=0;this.update_screen();var f=$(this.el).find(".upload-item");var e=this;f.each(function(){var g=$(this).find("#symbol");if(g.hasClass(e.state.queued)&&!g.hasClass(e.state.running)){g.removeClass(e.state.queued);g.addClass(e.state.init);$(this).find("#extension").attr("disabled",false);$(this).find("#space_to_tabs").attr("disabled",false)}})},event_reset:function(){if(this.counter.running==0){var e=$(this.el).find(".upload-item");$(this.el).find("table").fadeOut({complete:function(){e.remove()}});this.counter.reset();this.update_screen();this.uploadbox.reset()}},event_remove:function(e){var f=this.get_upload_item(e);var g=f.find("#symbol");if(g.hasClass(this.state.init)||g.hasClass(this.state.success)||g.hasClass(this.state.error)){if(f.hasClass("success")){this.counter.success--}else{if(f.hasClass("danger")){this.counter.error--}else{this.counter.announce--}}this.update_screen();this.uploadbox.remove(e);f.remove()}},event_show:function(g){g.preventDefault();if(!this.modal){var f=this;this.modal=new b.GalaxyModal({title:"Upload files from your local drive",body:this.template("upload-box","upload-info"),buttons:{Select:function(){f.uploadbox.select()},Upload:function(){f.event_upload()},Pause:function(){f.event_pause()},Reset:function(){f.event_reset()},Close:function(){f.modal.hide()}},height:"350"});this.setElement("#upload-box");var f=this;this.uploadbox=this.$el.uploadbox({dragover:function(){f.event_dragover()},dragleave:function(){f.event_dragleave()},announce:function(e,h,i){f.event_announce(e,h,i)},initialize:function(e,h,i){return f.event_initialize(e,h,i)},success:function(e,h,i){f.event_success(e,h,i)},progress:function(e,h,i){f.event_progress(e,h,i)},error:function(e,h,i){f.event_error(e,h,i)},complete:function(){f.event_complete()},});this.update_screen()}this.modal.show()},get_upload_item:function(e){return $(this.el).find("#upload-"+e)},size_to_string:function(e){var f="";if(e>=100000000000){e=e/100000000000;f="TB"}else{if(e>=100000000){e=e/100000000;f="GB"}else{if(e>=100000){e=e/100000;f="MB"}else{if(e>=100){e=e/100;f="KB"}else{e=e*10;f="b"}}}}return"<strong>"+(Math.round(e)/10)+"</strong> "+f},update_screen:function(){if(this.counter.announce==0){if(this.uploadbox.compatible){message="Drag&drop files into this box or click 'Select' to select files!"}else{message="Unfortunately, your browser does not support multiple file uploads or drag&drop.<br>Please upgrade to i.e. Firefox 4+, Chrome 7+, IE 10+, Opera 12+ or Safari 6+."}}else{if(this.counter.running==0){message="You added "+this.counter.announce+" file(s) to the queue. Add more files or click 'Upload' to proceed."}else{message="Please wait..."+this.counter.announce+" out of "+this.counter.running+" remaining."}}$("#upload-info").html(message);if(this.counter.running==0&&this.counter.announce+this.counter.success+this.counter.error>0){this.modal.enableButton("Reset")}else{this.modal.disableButton("Reset")}if(this.counter.running==0&&this.counter.announce>0){this.modal.enableButton("Upload")}else{this.modal.disableButton("Upload")}if(this.counter.running>0){this.modal.enableButton("Pause")}else{this.modal.disableButton("Pause")}if(this.counter.running==0){this.modal.enableButton("Select")}else{this.modal.disableButton("Select")}if(this.counter.announce+this.counter.success+this.counter.error>0){$(this.el).find("table").show()}else{$(this.el).find("table").hide()}},template:function(f,e){return'<div id="'+f+'" class="upload-box"><table class="table table-striped" style="display: none;"><thead><tr><th>Name</th><th>Size</th><th>Type</th><th>Space→Tab</th><th>Status</th><th></th></tr></thead><tbody></tbody></table></div><h6 id="'+e+'" class="upload-info"></h6>'},template_row:function(g,f){var e='<tr id="'+g.substr(1)+'" class="upload-item"><td><div id="title" class="title"></div></td><td><div id="size" class="size"></div></td><td><select id="extension" class="extension">';for(key in f){e+='<option value="'+key+'">'+f[key]+"</option>"}e+='</select></td><td><input id="space_to_tabs" type="checkbox"></input></td><td><div id="info" class="info"><div class="progress"><div class="progress-bar progress-bar-success"></div><div id="percentage" class="percentage">0%</div></div></div></td><td><div id="symbol" class="symbol '+this.state.init+'"></div></td></tr>';return e}});return{GalaxyUpload:a}}); \ No newline at end of file +define(["galaxy.modal","galaxy.master","utils/galaxy.utils","utils/galaxy.uploadbox","libs/backbone/backbone-relational"],function(b,d,c){var a=Backbone.View.extend({modal:null,button_show:null,uploadbox:null,select_extension:[["Auto-detect","auto"]],select_genome:[["Unspecified (?)","?"]],state:{init:"fa-icon-trash",queued:"fa-icon-spinner fa-icon-spin",running:"__running__",success:"fa-icon-ok",error:"fa-icon-warning-sign"},counter:{announce:0,success:0,error:0,running:0,reset:function(){this.announce=this.success=this.error=this.running=0}},initialize:function(){if(!Galaxy.currHistoryPanel){var e=this;window.setTimeout(function(){e.initialize()},500);return}if(!Galaxy.currUser.get("id")){return}var e=this;this.button_show=new d.GalaxyMasterIcon({icon:"fa-icon-upload",tooltip:"Upload Files",on_click:function(f){e.event_show(f)},on_unload:function(){if(e.counter.running>0){return"Several uploads are still processing."}},with_number:true});Galaxy.master.prepend(this.button_show);var e=this;c.jsonFromUrl(galaxy_config.root+"api/datatypes",function(f){for(key in f){e.select_extension.push([f[key],f[key]])}});c.jsonFromUrl(galaxy_config.root+"api/genomes",function(f){var g=e.select_genome[0];e.select_genome=[];for(key in f){if(f[key].length>1){if(f[key][1]!==g[1]){e.select_genome.push(f[key])}}}e.select_genome.sort(function(i,h){return i[0]>h[0]?1:i[0]<h[0]?-1:0});e.select_genome.unshift(g)})},event_dragover:function(f){},event_dragleave:function(f){},event_announce:function(i,g,n){var f="#upload-"+i;$(this.el).find("tbody:last").append(this.template_row(f));var h=this.get_upload_item(i);h.fadeIn();h.find("#title").html(g.name);h.find("#size").html(this.size_to_string(g.size));var m=this;h.find("#symbol").on("click",function(){m.event_remove(i)});h.find("#text-content").on("keyup",function(){var o=h.find("#text-content").val().length;h.find("#size").html(m.size_to_string(o))});this.event_progress(i,g,0);this.counter.announce++;this.update_screen();if(g.size==-1){var l=h.find("#text");var j=8;var e=h.width()-2*j;var k=h.height()-j;l.css("width",e+"px");l.css("top",k+"px");h.height(k+l.height()+2*j);l.show()}},event_initialize:function(i,e,n){this.button_show.number(this.counter.announce);var g=this.get_upload_item(i);var k=g.find("#symbol");k.addClass(this.state.running);var j=Galaxy.currHistoryPanel.model.get("id");var f=g.find("#extension").val();var m=g.find("#genome").val();var l=g.find("#text-content").val();var h=g.find("#space_to_tabs").is(":checked");if(!l&&!(e.size>0)){return null}this.uploadbox.configure({url:galaxy_config.root+"api/tools/",paramname:"files_0|file_data"});tool_input={};tool_input.dbkey=m;tool_input.file_type=f;tool_input["files_0|NAME"]=e.name;tool_input["files_0|type"]="upload_dataset";tool_input["files_0|url_paste"]=l;tool_input.space_to_tabs=h;data={};data.history_id=j;data.tool_id="upload1";data.inputs=JSON.stringify(tool_input);return data},event_progress:function(f,g,i){var h=this.get_upload_item(f);var e=parseInt(i);h.find(".progress-bar").css({width:e+"%"});if(e!=100){h.find("#percentage").html(e+"%")}else{h.find("#percentage").html("Adding to history...")}},event_success:function(e,f,h){this.event_progress(e,f,100);this.button_show.number("");this.counter.announce--;this.counter.success++;this.update_screen();var g=this.get_upload_item(e);g.addClass("success");g.find("#percentage").html("100%");var i=g.find("#symbol");i.removeClass(this.state.running);i.removeClass(this.state.queued);i.addClass(this.state.success);Galaxy.currHistoryPanel.refresh()},event_error:function(e,f,h){this.event_progress(e,f,0);this.button_show.number("");this.counter.announce--;this.counter.error++;this.update_screen();var g=this.get_upload_item(e);g.addClass("danger");g.find(".progress").remove();g.find("#info").html("<strong>Failed: </strong>"+h).show();var i=g.find("#symbol");i.removeClass(this.state.running);i.removeClass(this.state.queued);i.addClass(this.state.error)},event_start:function(){if(this.counter.announce==0||this.counter.running>0){return}var f=$(this.el).find(".upload-item");var e=this;f.each(function(){var g=$(this).find("#symbol");if(g.hasClass(e.state.init)){g.removeClass(e.state.init);g.addClass(e.state.queued);$(this).find("#text-content").attr("disabled",true);$(this).find("#genome").attr("disabled",true);$(this).find("#extension").attr("disabled",true);$(this).find("#space_to_tabs").attr("disabled",true)}});this.counter.running=this.counter.announce;this.update_screen();this.uploadbox.start()},event_stop:function(){if(this.counter.running==0){return}this.uploadbox.stop();$("#upload-info").html("Queue will pause after completing the current file...")},event_complete:function(){this.counter.running=0;this.update_screen();var f=$(this.el).find(".upload-item");var e=this;f.each(function(){var g=$(this).find("#symbol");if(g.hasClass(e.state.queued)&&!g.hasClass(e.state.running)){g.removeClass(e.state.queued);g.addClass(e.state.init);$(this).find("#text-content").attr("disabled",false);$(this).find("#genome").attr("disabled",false);$(this).find("#extension").attr("disabled",false);$(this).find("#space_to_tabs").attr("disabled",false)}})},event_reset:function(){if(this.counter.running==0){var e=$(this.el).find(".upload-item");$(this.el).find("table").fadeOut({complete:function(){e.remove()}});this.counter.reset();this.update_screen();this.uploadbox.reset()}},event_remove:function(e){var f=this.get_upload_item(e);var g=f.find("#symbol");if(g.hasClass(this.state.init)||g.hasClass(this.state.success)||g.hasClass(this.state.error)){if(f.hasClass("success")){this.counter.success--}else{if(f.hasClass("danger")){this.counter.error--}else{this.counter.announce--}}this.update_screen();this.uploadbox.remove(e);f.remove()}},event_create:function(){this.uploadbox.add([{name:"New File",size:-1}])},event_show:function(g){g.preventDefault();if(!this.modal){var f=this;this.modal=new b.GalaxyModal({title:"Upload files from your local drive",body:this.template("upload-box","upload-info"),buttons:{Select:function(){f.uploadbox.select()},Create:function(){f.event_create()},Upload:function(){f.event_start()},Pause:function(){f.event_stop()},Reset:function(){f.event_reset()},Close:function(){f.modal.hide()},},height:"400",width:"900"});this.setElement("#upload-box");var f=this;this.uploadbox=this.$el.uploadbox({dragover:function(){f.event_dragover()},dragleave:function(){f.event_dragleave()},announce:function(e,h,i){f.event_announce(e,h,i)},initialize:function(e,h,i){return f.event_initialize(e,h,i)},success:function(e,h,i){f.event_success(e,h,i)},progress:function(e,h,i){f.event_progress(e,h,i)},error:function(e,h,i){f.event_error(e,h,i)},complete:function(){f.event_complete()},});this.update_screen()}this.modal.show()},get_upload_item:function(e){return $(this.el).find("#upload-"+e)},size_to_string:function(e){var f="";if(e>=100000000000){e=e/100000000000;f="TB"}else{if(e>=100000000){e=e/100000000;f="GB"}else{if(e>=100000){e=e/100000;f="MB"}else{if(e>=100){e=e/100;f="KB"}else{if(e>0){e=e*10;f="b"}else{return"<strong>-</strong>"}}}}}return"<strong>"+(Math.round(e)/10)+"</strong> "+f},update_screen:function(){if(this.counter.announce==0){if(this.uploadbox.compatible()){message="Drag&drop files into this box or click 'Select' to select files!"}else{message="Unfortunately, your browser does not support multiple file uploads or drag&drop.<br>Please upgrade to i.e. Firefox 4+, Chrome 7+, IE 10+, Opera 12+ or Safari 6+."}}else{if(this.counter.running==0){message="You added "+this.counter.announce+" file(s) to the queue. Add more files or click 'Upload' to proceed."}else{message="Please wait..."+this.counter.announce+" out of "+this.counter.running+" remaining."}}$("#upload-info").html(message);if(this.counter.running==0&&this.counter.announce+this.counter.success+this.counter.error>0){this.modal.enableButton("Reset")}else{this.modal.disableButton("Reset")}if(this.counter.running==0&&this.counter.announce>0){this.modal.enableButton("Upload")}else{this.modal.disableButton("Upload")}if(this.counter.running>0){this.modal.enableButton("Pause")}else{this.modal.disableButton("Pause")}if(this.counter.running==0){this.modal.enableButton("Select");this.modal.enableButton("Create")}else{this.modal.disableButton("Select");this.modal.disableButton("Create")}if(this.counter.announce+this.counter.success+this.counter.error>0){$(this.el).find("table").show()}else{$(this.el).find("table").hide()}},template:function(f,e){return'<div id="'+f+'" class="upload-box"><table class="table table-striped" style="display: none;"><thead><tr><th>Name</th><th>Size</th><th>Type</th><th>Genome</th><th>Space→Tab</th><th>Status</th><th></th></tr></thead><tbody></tbody></table></div><h6 id="'+e+'" class="upload-info"></h6>'},template_row:function(f){var e='<tr id="'+f.substr(1)+'" class="upload-item"><td><div style="position: relative;"><div id="title" class="title"></div><div id="text" class="text"><div class="text-info">You may specify a list of URLs (one per line) or paste the contents of a file.</div><textarea id="text-content" class="text-content form-control"></textarea></div></div></td><td><div id="size" class="size"></div></td>';e+='<td><select id="extension" class="extension">';for(key in this.select_extension){e+='<option value="'+this.select_extension[key][1]+'">'+this.select_extension[key][0]+"</option>"}e+="</select></td>";e+='<td><select id="genome" class="genome">';for(key in this.select_genome){e+='<option value="'+this.select_genome[key][1]+'">'+this.select_genome[key][0]+"</option>"}e+="</select></td>";e+='<td><input id="space_to_tabs" type="checkbox"></input></td><td><div id="info" class="info"><div class="progress"><div class="progress-bar progress-bar-success"></div><div id="percentage" class="percentage">0%</div></div></div></td><td><div id="symbol" class="symbol '+this.state.init+'"></div></td></tr>';return e}});return{GalaxyUpload:a}}); \ No newline at end of file diff -r f888228f48d89ade8b8f49a4d777c573174392d1 -r 5c9cfd1dea8bfb85bcc1ee07035629a77ee3a051 static/scripts/packed/mvc/ui.js --- a/static/scripts/packed/mvc/ui.js +++ b/static/scripts/packed/mvc/ui.js @@ -1,1 +1,1 @@ -var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.attributes);a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(this.model.attributes.on_click){this.model.attributes.on_click(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){c._renderAndShow(d);return false});this.$button.data("PopupMenu",this)},render:function(){var a=this;this.$el.addClass("popmenu-wrapper").css({position:"absolute",display:"none"});this.$el.html(this.template(this.$button.attr("id"),this.options));if(this.options.length){this.$el.find("li").each(function(c,b){var f=$(b),e=f.children("a.popupmenu-option"),d=a.options[c].func;if(e.length&&d){e.click(function(g){d(g,a.options[c])})}a.options[c].$li=f})}return this},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_renderAndShow:function(a){this.render();this.$el.appendTo("body");this.$el.css(this._getShownPosition(a));this._setUpCloseBehavior();this.$el.show()},_setUpCloseBehavior:function(){var b=this;var a=function(c){c.bind("click.close_popup",function(){b.remove();c.unbind("click.close_popup")})};a($(window.document));a($(window.top.document));_.each(window.top.frames,function(c){a($(c.document))})},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"},template:function(e,b){var a='<ul id="'+e+'-menu" class="dropdown-menu">';if(b.length>0){for(var c in b){var d=b[c];if(d.divider){a+='<li class="divider"></li>'}else{if(d.header){a+='<li class="head"><a href="javascript:void(0);">'+d.html+"</a></li>"}else{if(d.href){a+='<li><a href="'+d.href+'"';a+='target="'+d.target+'"'}else{a+='<li><a href="javascript:void(0);"'}a+='class="popupmenu-option">';if(d.checked){a+='<span class="fa-icon-ok"></span>'}a+=d.html}}}}else{a+="<li>No Options.</li>"}return a+"</ul>"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(j){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}var i;switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;case"demo":if(i===undefined||i.closed){i=window.open(j,k);i.creator=self}break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a}; \ No newline at end of file +var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.attributes);a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(this.model.attributes.on_click){this.model.attributes.on_click(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body");this.$el.css(this._getShownPosition(a));this._setUpCloseBehavior();this.$el.show()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+e+'"'):(""),b=(d.checked)?('<span class="fa-icon-ok"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var b=this;var a=function(c){c.one("click.close_popup",function(){b.remove()})};a($(window.document));a($(window.top.document));_.each(window.top.frames,function(c){a($(c.document))})},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(j){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}var i;switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;case"demo":if(i===undefined||i.closed){i=window.open(j,k);i.creator=self}break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a}; \ No newline at end of file diff -r f888228f48d89ade8b8f49a4d777c573174392d1 -r 5c9cfd1dea8bfb85bcc1ee07035629a77ee3a051 static/scripts/packed/utils/galaxy.uploadbox.js --- a/static/scripts/packed/utils/galaxy.uploadbox.js +++ b/static/scripts/packed/utils/galaxy.uploadbox.js @@ -1,1 +1,1 @@ -(function(d){jQuery.event.props.push("dataTransfer");var h={url:"",paramname:"content",maxfilesize:2048,maxfilenumber:20,dragover:function(){},dragleave:function(){},announce:function(){},initialize:function(){},progress:function(){},success:function(){},error:function(j,k,l){alert(l)},complete:function(){},error_filesize:"File exceeds 2GB. Please use an FTP client.",error_default:"Please make sure the file is available.",error_server:"The server is unavailable.",error_toomany:"You can only queue <20 files per upload session."};var b={};var f={};var g=0;var i=0;var e=false;var a=false;var c=null;d.fn.uploadbox=function(z){b=d.extend({},h,z);c=this;c.append('<input id="uploadbox_input" type="file" style="display: none" multiple>');c.on("drop",m);c.on("dragover",n);c.on("dragleave",u);d("#uploadbox_input").change(function(A){w(A.target.files);d(this).val("")});function m(A){if(!A.dataTransfer){return}w(A.dataTransfer.files);A.preventDefault();return false}function n(A){A.preventDefault();b.dragover.call(A)}function u(A){A.stopPropagation();b.dragleave.call(A)}function j(A){if(A.lengthComputable){b.progress(this.index,this.file,Math.round((A.loaded*100)/A.total))}}function w(C){if(e){return}for(var B=0;B<C.length;B++){if(i>=b.maxfilenumber){break}var A=String(g++);f[A]=C[B];b.announce(A,f[A],"");i++}}function p(A){if(f[A]){delete f[A];i--}}function k(){if(i==0||a){a=false;e=false;b.complete();return}else{e=true}var D=-1;for(var F in f){D=F;break}var E=f[D];p(D);var H=b.initialize(D,E);try{var B=new FileReader();var C=E.size;var A=1048576*b.maxfilesize;B.index=D;if(C<A){B.onload=function(I){o(D,E,H)};B.onerror=function(I){s(D,E,b.error_default)};B.onabort=function(I){s(D,E,b.error_default)};B.readAsDataURL(E)}else{s(D,E,b.error_filesize)}}catch(G){s(D,E,G)}}function o(A,C,D){var E=new FormData();for(var B in D){E.append(B,D[B])}E.append(b.paramname,C,C.name);var F=new XMLHttpRequest();F.onreadystatechange=function(){if(F.readyState!=F.DONE){return}var G=null;if(F.responseText){try{G=jQuery.parseJSON(F.responseText)}catch(H){G=F.responseText}}if(F.status<200||F.status>299){var I=F.statusText;if(!F.statusText){if(F.status==0){I=b.error_server}else{I=b.error_default}}s(A,C,I+" ("+F.status+")")}else{v(A,C,G)}};F.upload.index=A;F.upload.file=C;F.upload.addEventListener("progress",j,false);F.open("POST",b.url,true);F.setRequestHeader("Accept","application/json");F.setRequestHeader("Cache-Control","no-cache");F.setRequestHeader("X-Requested-With","XMLHttpRequest");F.send(E)}function v(A,B,C){b.success(A,B,C);k()}function s(A,B,C){b.error(A,B,C);k()}function t(){d("#uploadbox_input").trigger("click")}function r(A){for(A in f){p(A)}}function y(){if(!e){e=true;k()}}function x(){a=true}function l(A){b=d.extend({},b,A);return b}function q(){return window.File&&window.FileReader&&window.FormData&&window.XMLHttpRequest&&window.FileList}return{select:t,remove:p,upload:y,pause:x,reset:r,configure:l,compatible:q}}})(jQuery); \ No newline at end of file +(function(c){jQuery.event.props.push("dataTransfer");var h={url:"",paramname:"content",maxfilesize:2048,maxfilenumber:20,dragover:function(){},dragleave:function(){},announce:function(){},initialize:function(){},progress:function(){},success:function(){},error:function(k,l,m){alert(m)},complete:function(){},error_filesize:"File exceeds 2GB. Please use an FTP client.",error_default:"Please make sure the file is available.",error_server:"Upload request failed.",error_toomany:"You can only queue <20 files per upload session.",error_login:"Uploads require you to log in.",error_missing:"No upload content available."};var a={};var e={};var f=0;var j=0;var d=false;var g=false;var b=null;var i=null;c.fn.uploadbox=function(A){a=c.extend({},h,A);b=this;b.append('<input id="uploadbox_input" type="file" style="display: none" multiple>');b.on("drop",o);b.on("dragover",p);b.on("dragleave",x);c("#uploadbox_input").change(function(B){z(B.target.files);c(this).val("")});function o(B){if(!B.dataTransfer){return}z(B.dataTransfer.files);B.preventDefault();return false}function p(B){B.preventDefault();a.dragover.call(B)}function x(B){B.stopPropagation();a.dragleave.call(B)}function k(B){if(B.lengthComputable){a.progress(this.index,this.file,Math.round((B.loaded*100)/B.total))}}function z(D){if(d){return}for(var C=0;C<D.length;C++){if(j>=a.maxfilenumber){break}var B=String(f++);e[B]=D[C];a.announce(B,e[B],"");j++}}function r(B){if(e[B]){delete e[B];j--}}function m(){if(j==0||g){g=false;d=false;a.complete();return}else{d=true}var D=-1;for(var F in e){D=F;break}var E=e[D];r(D);var C=E.size;var B=1048576*a.maxfilesize;if(C<B){var G=a.initialize(D,E);if(G){q(D,E,G)}else{u(D,E,a.error_missing)}}else{u(D,E,a.error_filesize)}}function q(B,D,E){var F=new FormData();for(var C in E){F.append(C,E[C])}if(D.size>0){F.append(a.paramname,D,D.name)}i=new XMLHttpRequest();i.open("POST",a.url,true);i.setRequestHeader("Accept","application/json");i.setRequestHeader("Cache-Control","no-cache");i.setRequestHeader("X-Requested-With","XMLHttpRequest");i.onreadystatechange=function(){if(i.readyState!=i.DONE){return}var G=null;if(i.responseText){try{G=jQuery.parseJSON(i.responseText)}catch(H){G=i.responseText}}if(i.status<200||i.status>299){var I=i.statusText;if(i.status==403){I=a.error_login}else{if(i.status==0){I=a.error_server}else{if(!I){I=a.error_default}}}u(B,D,I+" ("+i.status+")")}else{y(B,D,G)}};i.upload.index=B;i.upload.file=D;i.upload.addEventListener("progress",k,false);i.send(F)}function y(B,C,D){a.success(B,C,D);m()}function u(B,C,D){a.error(B,C,D);m()}function w(){c("#uploadbox_input").trigger("click")}function t(B){for(B in e){r(B)}}function l(){if(!d){d=true;m()}}function v(){g=true}function n(B){a=c.extend({},a,B);return a}function s(){return window.File&&window.FormData&&window.XMLHttpRequest&&window.FileList}return{select:w,add:z,remove:r,start:l,stop:v,reset:t,configure:n,compatible:s}}})(jQuery); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d816d6602e18/ Changeset: d816d6602e18 Branch: custom-toolbox2 User: BjoernGruening Date: 2013-06-13 22:46:22 Summary: Implement the ability to change the tool-panel as user prederence. Affected #: 10 files diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -6,8 +6,9 @@ import logging, logging.config import ConfigParser from datetime import timedelta -from galaxy.util import string_as_bool, listify, parse_xml - +from galaxy.util import string_as_bool +from galaxy.util import listify +from galaxy.util import parse_xml from galaxy import eggs import pkg_resources @@ -64,9 +65,14 @@ tcf = kwargs[ 'tool_config_files' ] else: tcf = 'tool_conf.xml' - self.tool_filters = listify( kwargs.get( "tool_filters", [] ) ) - self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ) ) - self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ) ) + self.tool_filters = listify( kwargs.get( "tool_filters", [] ), do_strip=True ) + self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ), do_strip=True ) + self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ), do_strip=True ) + + self.user_tool_filters = listify( kwargs.get( "user_tool_filters", [] ), do_strip=True ) + self.user_label_filters = listify( kwargs.get( "user_tool_label_filters", [] ), do_strip=True ) + self.user_section_filters = listify( kwargs.get( "user_tool_section_filters", [] ), do_strip=True ) + self.tool_configs = [ resolve_path( p, self.root ) for p in listify( tcf ) ] self.shed_tool_data_path = kwargs.get( "shed_tool_data_path", None ) if self.shed_tool_data_path: @@ -459,3 +465,4 @@ sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) root.addHandler( sentry_handler ) + diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3267,7 +3267,7 @@ #end of Data Manager Classes class UserPreference ( object ): - def __init__( self, name=None, value=None ): + def __init__( self, name=None, value=None): self.name = name self.value = value diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3426,3 +3426,4 @@ class InterruptedUpload( Exception ): pass + diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf lib/galaxy/tools/filters/__init__.py --- a/lib/galaxy/tools/filters/__init__.py +++ b/lib/galaxy/tools/filters/__init__.py @@ -1,4 +1,5 @@ - +from galaxy.util import listify +from copy import deepcopy class FilterFactory( object ): """ @@ -15,17 +16,29 @@ self.default_filters = dict( tool=[ _not_hidden ], section=[], label=[] ) # Add dynamic filters to these default filters. config = toolbox.app.config - self.__init_filters( "tool", config.tool_filters ) - self.__init_filters( "section", config.tool_section_filters ) - self.__init_filters( "label", config.tool_label_filters ) + self.__init_filters( "tool", config.tool_filters, self.default_filters ) + self.__init_filters( "section", config.tool_section_filters, self.default_filters ) + self.__init_filters( "label", config.tool_label_filters, self.default_filters ) def build_filters( self, trans, **kwds ): """ Build list of filters to check tools against given current context. """ - filters = self.default_filters.copy() - - if not trans.user: + filters = deepcopy( self.default_filters ) + if trans.user: + for name, value in trans.user.preferences.items(): + if value.strip(): + user_filters = listify( value, do_strip=True ) + category = '' + if name == 'toolbox_tool_filters': + category = "tool" + elif name == 'toolbox_section_filters': + category = "section" + elif name == 'toolbox_label_filters': + category = "label" + if category: + self.__init_filters( category, user_filters, filters ) + else: filters[ "tool" ].append( _requires_login ) if kwds.get( "trackster", False ): @@ -33,10 +46,11 @@ return filters - def __init_filters( self, key, filters ): + def __init_filters( self, key, filters, toolbox_filters ): for filter in filters: filter_function = self.__build_filter_function( filter ) - self.default_filters[ key ].append( filter_function ) + toolbox_filters[ key ].append( filter_function ) + return toolbox_filters def __build_filter_function( self, filter_name ): """Obtain python function (importing a submodule if needed) diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf lib/galaxy/tools/filters/examples.py --- /dev/null +++ b/lib/galaxy/tools/filters/examples.py @@ -0,0 +1,115 @@ +import logging +log = logging.getLogger( __name__ ) + + +def restrict_encode( content, tool ): + """ + Disable the random interval ENCODE tool + + This tool filter will disable all the ENCODE tool when enabled. + """ + if tool.id == 'random_intervals1': + return False + return True + + +def restrict_text( content, section ): + """ + Disable Text sections + + This tool filter will disable all Tools groups under a 'Text' section when enabled. + """ + if section.name.find('Text') != -1: + return False + return True + + +def restrict_upload_to_admins( context, tool ): + """ + Disable Upload tool for all non-admin users. + + This tool filter will hide the upload tool from all users except admin + users. + + This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:restrict_upload_to_admins + """ + if tool.name == "Upload File": + return context.trans.user_is_admin() + return True + + +def disable_gatk( context, tool ): + """ + This tool filter will disable all gatk tools when enabled. + + This can be enabled by renaming this file to examples.py and adding the following to the + ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:disable_gatk + """ + return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) + + +def explicit_user_mapping( context, section ): + """ + This tool section filter uses an explicit mapping to describe what users can view + which tool sections. Anonymous users will only be able to view the "Get Data" + tool section (with id getext). This can be enabled by renaming this file to + examples.py and adding the following to the ``app:main`` section of + ``universe_wsgi.ini``: + + tool_section_filters = examples:explicit_user_mapping + """ + users_sections = { + None: [ "getext" ], + "bob(a)example.com": [ "getext", "textutil", "filter" ], + "mary(a)example.com": [ "getext", "textutil", "filter", "ngs" ], + } + user = context.trans.user + email = user and user.email + valid_sections = users_sections.get( email, [] ) + return section.id in valid_sections + + +DEVELOPERS = [ "mary(a)example.com" ] + + +def restrict_development_tools( context, tool ): + """ + This tool filter will disable all tools with the string alpha appearing in + the version for all users except those explicitly appearing in the DEVELOPERS list + defined above. This can be enabled by renaming this file to examples.py and + adding the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:restrict_development_tools + """ + version = tool.version + user = context.trans.user + email = user and user.email + return "alpha" not in version or email in DEVELOPERS + + +def per_host_tool_sections( context, section ): + """ + This tool section filter results in different sections being display based on + the URL the user is making the request to. This could allow a single Galaxy instance + to seem like several different instances hosting different tools based on the URL used + to access the Galxy. This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_section_filters = examples:per_host_tool_sections + """ + host = context.trans.request.host + # Core tools used by all virtual hosts. + valid_sections = [ "getext", "textutil", "filter" ] + if "ngs.galaxy.example.com" in host: + valid_sections += [ "ngs" ] + elif "microarray.galaxy.example.com" in host: + valid_sections += [ "microarray" ] + elif "proteomics.galaxy.example.com" in host: + valid_sections += [ "proteomics" ] + return section.id in valid_sections + diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -496,7 +496,7 @@ else: return False -def listify( item ): +def listify( item, do_strip=False ): """ Make a single item a single item list, or return a list if passed a list. Passing a None returns an empty list. @@ -506,7 +506,10 @@ elif isinstance( item, list ): return item elif isinstance( item, basestring ) and item.count( ',' ): - return item.split( ',' ) + if do_strip: + return [token.strip() for token in item.split( ',' )] + else: + return item.split( ',' ) else: return [ item ] @@ -746,6 +749,32 @@ current_group, e ) ) +def docstring_trim(docstring): + """Trimming python doc strings. Taken from: http://www.python.org/dev/peps/pep-0257/""" + if not docstring: + return '' + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + indent = sys.maxint + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxint: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + def nice_size(size): """ Returns a readably formatted string with the size diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -9,14 +9,24 @@ import string import random from galaxy import web -from galaxy import util, model +from galaxy import util +from galaxy import model from galaxy.model.orm import and_ -from galaxy.security.validate_user_input import validate_email, validate_publicname, validate_password, transform_publicname -from galaxy.util.json import from_json_string, to_json_string +from galaxy.security.validate_user_input import validate_email +from galaxy.security.validate_user_input import validate_publicname +from galaxy.security.validate_user_input import validate_password +from galaxy.security.validate_user_input import transform_publicname +from galaxy.util.json import from_json_string +from galaxy.util.json import to_json_string +from galaxy.util import listify +from galaxy.util import docstring_trim from galaxy.web import url_for -from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin -from galaxy.web.form_builder import CheckboxField, build_select_field +from galaxy.web.base.controller import BaseUIController +from galaxy.web.base.controller import UsesFormDefinitionsMixin +from galaxy.web.form_builder import CheckboxField +from galaxy.web.form_builder import build_select_field from galaxy.web.framework.helpers import time_ago, grids +from galaxy.web.framework.helpers import grids log = logging.getLogger( __name__ ) @@ -101,6 +111,7 @@ use_panels=use_panels, message=message, status='error' ) ) + @web.expose def openid_process( self, trans, **kwd ): '''Handle's response from OpenID Providers''' @@ -221,6 +232,7 @@ redirect=redirect, message=message, status=status ) ) + @web.expose def openid_associate( self, trans, cntrller='user', **kwd ): '''Associates a user with an OpenID log in''' @@ -359,6 +371,7 @@ user_type_form_definition=user_type_form_definition, widgets=widgets, openids=openids ) + @web.expose @web.require_login( 'manage OpenIDs' ) def openid_disassociate( self, trans, **kwd ): @@ -465,6 +478,7 @@ openid_providers=trans.app.openid_providers, form_input_auto_focus=True, active_view="user" ) + def __validate_login( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) @@ -662,6 +676,7 @@ message = 'Now logged in as %s.<br><a target="_top" href="%s">Return to the home page.</a>' % ( user.email, url_for( '/' ) ) success = True return ( message, status, user, success ) + def __get_user_type_form_definition( self, trans, user=None, **kwd ): params = util.Params( kwd ) if user and user.values: @@ -673,6 +688,7 @@ else: user_type_form_definition = None return user_type_form_definition + def __get_widgets( self, trans, user_type_form_definition, user=None, **kwd ): widgets = [] if user_type_form_definition: @@ -774,6 +790,7 @@ username=user.username, message=message, status=status ) + @web.expose def edit_info( self, trans, cntrller, **kwd ): params = util.Params( kwd ) @@ -882,6 +899,7 @@ action='manage_user_info', cntrller=cntrller, **kwd ) ) + @web.expose def reset_password( self, trans, email=None, **kwd ): if trans.app.config.smtp_server is None: @@ -929,6 +947,7 @@ return trans.fill_template( '/user/reset_password.mako', message=message, status=status ) + def __validate( self, trans, params, email, password, confirm, username ): # If coming from the tool shed webapp, we'll require a public user name if trans.webapp.name == 'tool_shed': @@ -950,6 +969,7 @@ if user_type_fd_id in [ 'none' ]: return "Select the user's type and information" return message + @web.expose def set_default_permissions( self, trans, cntrller, **kwd ): """Sets the user's default permissions for the new histories""" @@ -975,7 +995,128 @@ status=status ) else: # User not logged in, history group must be only public - return trans.show_error_message( "You must be logged in to change your default permitted actions." ) + return trans.show_error_message( "You must be logged in to change your default permitted actions." ) + + @web.expose + @web.require_login() + def toolbox_filters( self, trans, cntrller, **kwd ): + """ + Sets the user's default filters for the toolbox. + Toolbox filters are specified in universe_wsgi.ini. + The user can activate them and the choice is stored in user_preferences. + """ + + def get_filter_mapping( db_filters, config_filters ): + """ + Compare the allowed filters from the universe_wsgi.ini config file with the previously saved or default filters from the database. + We need that to toogle the checkboxes for the formular in the right way. + Furthermore we extract all information associated to a filter to display them in the formular. + """ + filters = list() + for filter_name in config_filters: + if ":" in filter_name: + # Should be a submodule of filters (e.g. examples:restrict_development_tools) + (module_name, function_name) = filter_name.rsplit(":", 1) + module_name = 'galaxy.tools.filters.%s' % module_name.strip() + module = __import__( module_name, globals(), fromlist=['temp_module'] ) + function = getattr( module, function_name.strip() ) + else: + # No module found it has to be explicitly imported. + module = __import__( 'galaxy.tools.filters', globals(), fromlist=['temp_module'] ) + function = getattr( globals(), filter_name.strip() ) + + doc_string = docstring_trim( function.__doc__ ) + split = doc_string.split('\n\n') + if split: + sdesc = split[0] + else: + log.error( 'No description specified in the __doc__ string for %s.' % filter_name ) + if len(split) > 1: + description = split[1] + else: + description = '' + + if filter_name in db_filters: + filters.append( dict( filterpath=filter_name, short_desc=sdesc, desc=description, checked=True ) ) + else: + filters.append( dict( filterpath=filter_name, short_desc=sdesc, desc=description, checked=False ) ) + return filters + + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + + user_id = params.get( 'user_id', False ) + if user_id: + user = trans.sa_session.query( trans.app.model.User ).get( trans.security.decode_id( user_id ) ) + else: + user = trans.user + + if user: + saved_user_tool_filters = list() + saved_user_section_filters = list() + saved_user_label_filters = list() + + for name, value in user.preferences.items(): + if name == 'toolbox_tool_filters': + saved_user_tool_filters = listify( value, do_strip=True ) + elif name == 'toolbox_section_filters': + saved_user_section_filters = listify( value, do_strip=True ) + elif name == 'toolbox_label_filters': + saved_user_label_filters = listify( value, do_strip=True ) + + tool_filters = get_filter_mapping( saved_user_tool_filters, trans.app.config.user_tool_filters ) + section_filters = get_filter_mapping( saved_user_section_filters, trans.app.config.user_section_filters ) + label_filters = get_filter_mapping( saved_user_label_filters, trans.app.config.user_label_filters ) + + return trans.fill_template( 'user/toolbox_filters.mako', + cntrller=cntrller, + message=message, + tool_filters=tool_filters, + section_filters=section_filters, + label_filters=label_filters, + user=user, + status=status ) + else: + # User not logged in, history group must be only public + return trans.show_error_message( "You must be logged in to change private toolbox filters." ) + + @web.expose + @web.require_login( "to change the private toolbox filters" ) + def edit_toolbox_filters( self, trans, cntrller, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + user_id = params.get( 'user_id', False ) + if not user_id: + # User must be logged in to create a new address + return trans.show_error_message( "You must be logged in to change the ToolBox filters." ) + + user = trans.sa_session.query( trans.app.model.User ).get( trans.security.decode_id( user_id ) ) + + if params.get( 'edit_toolbox_filter_button', False ): + tool_filters = list() + section_filters = list() + label_filters = list() + for name, state in params.flatten(): + if state == 'on': + if name.startswith('t_'): + tool_filters.append( name[2:] ) + elif name.startswith('l_'): + label_filters.append( name[2:] ) + elif name.startswith('s_'): + section_filters.append( name[2:] ) + user.preferences['toolbox_tool_filters'] = ','.join( tool_filters ) + user.preferences['toolbox_section_filters'] = ','.join( section_filters ) + user.preferences['toolbox_label_filters'] = ','.join( label_filters ) + + trans.sa_session.add( user ) + trans.sa_session.flush() + message = 'ToolBox filters has been updated.' + kwd = dict( message=message, status='done' ) + + # Display the ToolBox filters form with the current values filled in + return self.toolbox_filters( trans, cntrller, **kwd ) + @web.expose @web.require_login( "to get most recently used tool" ) @web.json_pretty @@ -998,7 +1139,8 @@ "minsizehint" : tool.uihints.get( 'minwidth', -1 ), "description" : tool.description } - return tool_info + return tool_info + @web.expose def manage_addresses(self, trans, **kwd): if trans.user: @@ -1020,6 +1162,7 @@ else: # User not logged in, history group must be only public return trans.show_error_message( "You must be logged in to change your default permitted actions." ) + @web.expose def new_address( self, trans, cntrller, **kwd ): params = util.Params( kwd ) @@ -1104,6 +1247,7 @@ phone=phone, message=message, status=status ) + @web.expose def edit_address( self, trans, cntrller, **kwd ): params = util.Params( kwd ) @@ -1118,7 +1262,7 @@ address_id = params.get( 'address_id', None ) if not address_id: return trans.show_error_message( "No address id received for editing." ) - address_obj = trans.sa_session.query( trans.app.model.UserAddress ).get( trans.security.decode_id( address_id ) ) + address_obj = trans.sa_session.query( trans.app.model.UserAddress ).get( trans.security.decode_id( address_id ) ) if params.get( 'edit_address_button', False ): short_desc = util.restore_text( params.get( 'short_desc', '' ) ) name = util.restore_text( params.get( 'name', '' ) ) @@ -1181,6 +1325,7 @@ address_obj=address_obj, message=message, status=status ) + @web.expose def delete_address( self, trans, cntrller, address_id=None, user_id=None ): try: @@ -1201,6 +1346,7 @@ user_id=user_id, message=message, status=status ) ) + @web.expose def undelete_address( self, trans, cntrller, address_id=None, user_id=None ): try: @@ -1220,6 +1366,7 @@ user_id=user_id, message=message, status=status ) ) + @web.expose def set_user_pref_async( self, trans, pref_name, pref_value ): """ Set a user preference asynchronously. If user is not logged in, do nothing. """ @@ -1232,7 +1379,7 @@ """ Log a user action asynchronously. If user is not logged in, do nothing. """ if trans.user: trans.log_action( trans.get_user(), action, context, params ) - + @web.expose @web.require_login() def dbkeys( self, trans, **kwds ): diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf templates/user/index.mako --- a/templates/user/index.mako +++ b/templates/user/index.mako @@ -13,6 +13,7 @@ <li><a href="${h.url_for( controller='user', action='manage_user_info', cntrller=cntrller )}">${_('Manage your information')}</a></li><li><a href="${h.url_for( controller='user', action='set_default_permissions', cntrller=cntrller )}">${_('Change default permissions')}</a> for new histories</li><li><a href="${h.url_for( controller='user', action='api_keys', cntrller=cntrller )}">${_('Manage your API keys')}</a></li> + <li><a href="${h.url_for( controller='user', action='toolbox_filters', cntrller=cntrller )}">${_('Manage your ToolBox filters')}</a></li> %if trans.app.config.enable_openid: <li><a href="${h.url_for( controller='user', action='openid_manage', cntrller=cntrller )}">${_('Manage OpenIDs')}</a> linked to your account</li> %endif diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf templates/user/toolbox_filters.mako --- /dev/null +++ b/templates/user/toolbox_filters.mako @@ -0,0 +1,91 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, status )} +%endif +</br> +</br> + +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( controller='user', action='index', cntrller=cntrller )}">User preferences</a> + </li> +</ul> + +%if tool_filters or section_filters or label_filters: + <div class="toolForm"> + <form name="toolbox_filter" id="toolbox_filter" action="${h.url_for( controller='user', action='edit_toolbox_filters', cntrller=cntrller, user_id=trans.security.encode_id( user.id ) )}" method="post" > + % if tool_filters: + <div class="toolFormTitle">Edit ToolBox filters :: Tools</div> + <div class="toolFormBody"> + % for filter in tool_filters: + <div class="form-row"> + <div style="float: left; width: 40px; margin-right: 10px;"> + % if filter['checked']: + <input type="checkbox" name="t_${filter['filterpath']}" checked="checked"> + % else: + <input type="checkbox" name="t_${filter['filterpath']}"> + % endif + </div> + <div style="float: left; margin-right: 10px;"> + ${filter['short_desc']} + <div class="toolParamHelp" style="clear: both;">${filter['desc']}</div> + </div> + <div style="clear: both"></div> + </div> + % endfor + </div> + % endif + + % if section_filters: + <div class="toolFormTitle">Edit ToolBox filters :: Sections</div> + <div class="toolFormBody"> + % for filter in section_filters: + <div class="form-row"> + <div style="float: left; width: 40px; margin-right: 10px;"> + % if filter['checked']: + <input type="checkbox" name="s_${filter['filterpath']}" checked="checked"> + % else: + <input type="checkbox" name="s_${filter['filterpath']}"> + % endif + </div> + <div style="float: left; margin-right: 10px;"> + ${filter['short_desc']} + <div class="toolParamHelp" style="clear: both;">${filter['desc']}</div> + </div> + <div style="clear: both"></div> + </div> + % endfor + </div> + % endif + + % if label_filters: + <div class="toolFormTitle">Edit ToolBox filters :: Labels</div> + <div class="toolFormBody"> + % for filter in label_filters: + <div class="form-row"> + <div style="float: left; width: 40px; margin-right: 10px;"> + % if filter['checked']: + <input type="checkbox" name="l_${filter['filterpath']}" checked="checked"> + % else: + <input type="checkbox" name="l_${filter['filterpath']}"> + % endif + </div> + <div style="float: left; margin-right: 10px;"> + ${filter['short_desc']} + <div class="toolParamHelp" style="clear: both;">${filter['desc']}</div> + </div> + <div style="clear: both"></div> + </div> + % endfor + </div> + % endif + <div class="form-row"> + <input type="submit" name="edit_toolbox_filter_button" value="Save changes"> + </div> + </form> + </div> +%else: + ${render_msg( 'No filter available. Contact you system administrator or check your configuration file.', 'info' )} +%endif diff -r 6f9e73ec3e99d43f14e3d735ecfe7b66ea38472a -r d816d6602e18fd8e04c5d141527ca77a3e39e6cf universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -682,6 +682,18 @@ # if running many handlers. #cache_user_job_count = False +# ToolBox filtering +# Modules from lib/galaxy/tools/filters/ can be specified in the following lines. +# tool_* filters will be applied for all users and can not be changed by them. +# user_tool_* filters will be shown under user preferences and can be toogled +# on and off by runtime +#tool_filters +#tool_label_filters +#tool_section_filters +#user_tool_filters = examples:restrict_upload_to_admins, examples:restrict_encode +#user_tool_section_filters = examples:restrict_text +#user_tool_label_filters = examples:restrict_upload_to_admins, examples:restrict_encode + # ---- Galaxy Message Queue ------------------------------------------------- # Galaxy uses AMQ protocol to receive messages from external sources like https://bitbucket.org/galaxy/galaxy-central/commits/fb1b676f51b2/ Changeset: fb1b676f51b2 User: dannon Date: 2013-10-17 19:40:04 Summary: Merged custom-toolbox2 Affected #: 10 files diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -6,8 +6,9 @@ import logging, logging.config import ConfigParser from datetime import timedelta -from galaxy.util import string_as_bool, listify, parse_xml - +from galaxy.util import string_as_bool +from galaxy.util import listify +from galaxy.util import parse_xml from galaxy import eggs import pkg_resources @@ -64,9 +65,14 @@ tcf = kwargs[ 'tool_config_files' ] else: tcf = 'tool_conf.xml,shed_tool_conf.xml' - self.tool_filters = listify( kwargs.get( "tool_filters", [] ) ) - self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ) ) - self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ) ) + self.tool_filters = listify( kwargs.get( "tool_filters", [] ), do_strip=True ) + self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ), do_strip=True ) + self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ), do_strip=True ) + + self.user_tool_filters = listify( kwargs.get( "user_tool_filters", [] ), do_strip=True ) + self.user_label_filters = listify( kwargs.get( "user_tool_label_filters", [] ), do_strip=True ) + self.user_section_filters = listify( kwargs.get( "user_tool_section_filters", [] ), do_strip=True ) + self.tool_configs = [ resolve_path( p, self.root ) for p in listify( tcf ) ] self.shed_tool_data_path = kwargs.get( "shed_tool_data_path", None ) if self.shed_tool_data_path: @@ -489,3 +495,4 @@ sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) root.addHandler( sentry_handler ) + diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3359,7 +3359,7 @@ #end of Data Manager Classes class UserPreference ( object ): - def __init__( self, name=None, value=None ): + def __init__( self, name=None, value=None): self.name = name self.value = value diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3433,3 +3433,4 @@ class InterruptedUpload( Exception ): pass + diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 lib/galaxy/tools/filters/__init__.py --- a/lib/galaxy/tools/filters/__init__.py +++ b/lib/galaxy/tools/filters/__init__.py @@ -1,4 +1,5 @@ - +from galaxy.util import listify +from copy import deepcopy class FilterFactory( object ): """ @@ -15,25 +16,39 @@ self.default_filters = dict( tool=[ _not_hidden, _handle_requires_login ], section=[], label=[] ) # Add dynamic filters to these default filters. config = toolbox.app.config - self.__init_filters( "tool", config.tool_filters ) - self.__init_filters( "section", config.tool_section_filters ) - self.__init_filters( "label", config.tool_label_filters ) + self.__init_filters( "tool", config.tool_filters, self.default_filters ) + self.__init_filters( "section", config.tool_section_filters, self.default_filters ) + self.__init_filters( "label", config.tool_label_filters, self.default_filters ) def build_filters( self, trans, **kwds ): """ Build list of filters to check tools against given current context. """ - filters = self.default_filters.copy() - + filters = deepcopy( self.default_filters ) + if trans.user: + for name, value in trans.user.preferences.items(): + if value.strip(): + user_filters = listify( value, do_strip=True ) + category = '' + if name == 'toolbox_tool_filters': + category = "tool" + elif name == 'toolbox_section_filters': + category = "section" + elif name == 'toolbox_label_filters': + category = "label" + if category: + self.__init_filters( category, user_filters, filters ) + else: if kwds.get( "trackster", False ): filters[ "tool" ].append( _has_trackster_conf ) return filters - def __init_filters( self, key, filters ): + def __init_filters( self, key, filters, toolbox_filters ): for filter in filters: filter_function = self.__build_filter_function( filter ) - self.default_filters[ key ].append( filter_function ) + toolbox_filters[ key ].append( filter_function ) + return toolbox_filters def __build_filter_function( self, filter_name ): """Obtain python function (importing a submodule if needed) diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 lib/galaxy/tools/filters/examples.py --- /dev/null +++ b/lib/galaxy/tools/filters/examples.py @@ -0,0 +1,115 @@ +import logging +log = logging.getLogger( __name__ ) + + +def restrict_encode( content, tool ): + """ + Disable the random interval ENCODE tool + + This tool filter will disable all the ENCODE tool when enabled. + """ + if tool.id == 'random_intervals1': + return False + return True + + +def restrict_text( content, section ): + """ + Disable Text sections + + This tool filter will disable all Tools groups under a 'Text' section when enabled. + """ + if section.name.find('Text') != -1: + return False + return True + + +def restrict_upload_to_admins( context, tool ): + """ + Disable Upload tool for all non-admin users. + + This tool filter will hide the upload tool from all users except admin + users. + + This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:restrict_upload_to_admins + """ + if tool.name == "Upload File": + return context.trans.user_is_admin() + return True + + +def disable_gatk( context, tool ): + """ + This tool filter will disable all gatk tools when enabled. + + This can be enabled by renaming this file to examples.py and adding the following to the + ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:disable_gatk + """ + return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) + + +def explicit_user_mapping( context, section ): + """ + This tool section filter uses an explicit mapping to describe what users can view + which tool sections. Anonymous users will only be able to view the "Get Data" + tool section (with id getext). This can be enabled by renaming this file to + examples.py and adding the following to the ``app:main`` section of + ``universe_wsgi.ini``: + + tool_section_filters = examples:explicit_user_mapping + """ + users_sections = { + None: [ "getext" ], + "bob(a)example.com": [ "getext", "textutil", "filter" ], + "mary(a)example.com": [ "getext", "textutil", "filter", "ngs" ], + } + user = context.trans.user + email = user and user.email + valid_sections = users_sections.get( email, [] ) + return section.id in valid_sections + + +DEVELOPERS = [ "mary(a)example.com" ] + + +def restrict_development_tools( context, tool ): + """ + This tool filter will disable all tools with the string alpha appearing in + the version for all users except those explicitly appearing in the DEVELOPERS list + defined above. This can be enabled by renaming this file to examples.py and + adding the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:restrict_development_tools + """ + version = tool.version + user = context.trans.user + email = user and user.email + return "alpha" not in version or email in DEVELOPERS + + +def per_host_tool_sections( context, section ): + """ + This tool section filter results in different sections being display based on + the URL the user is making the request to. This could allow a single Galaxy instance + to seem like several different instances hosting different tools based on the URL used + to access the Galxy. This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_section_filters = examples:per_host_tool_sections + """ + host = context.trans.request.host + # Core tools used by all virtual hosts. + valid_sections = [ "getext", "textutil", "filter" ] + if "ngs.galaxy.example.com" in host: + valid_sections += [ "ngs" ] + elif "microarray.galaxy.example.com" in host: + valid_sections += [ "microarray" ] + elif "proteomics.galaxy.example.com" in host: + valid_sections += [ "proteomics" ] + return section.id in valid_sections + diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -504,7 +504,7 @@ else: return False -def listify( item ): +def listify( item, do_strip=False ): """ Make a single item a single item list, or return a list if passed a list. Passing a None returns an empty list. @@ -514,7 +514,10 @@ elif isinstance( item, list ): return item elif isinstance( item, basestring ) and item.count( ',' ): - return item.split( ',' ) + if do_strip: + return [token.strip() for token in item.split( ',' )] + else: + return item.split( ',' ) else: return [ item ] @@ -778,6 +781,32 @@ current_group, e ) ) +def docstring_trim(docstring): + """Trimming python doc strings. Taken from: http://www.python.org/dev/peps/pep-0257/""" + if not docstring: + return '' + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + indent = sys.maxint + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxint: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + def nice_size(size): """ Returns a readably formatted string with the size diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -10,15 +10,25 @@ import random import urllib from galaxy import web -from galaxy import util, model +from galaxy import util +from galaxy import model from galaxy.model.orm import and_ -from galaxy.security.validate_user_input import validate_email, validate_publicname, validate_password, transform_publicname -from galaxy.util.json import from_json_string, to_json_string +from galaxy.security.validate_user_input import validate_email +from galaxy.security.validate_user_input import validate_publicname +from galaxy.security.validate_user_input import validate_password +from galaxy.security.validate_user_input import transform_publicname +from galaxy.util.json import from_json_string +from galaxy.util.json import to_json_string +from galaxy.util import listify +from galaxy.util import docstring_trim from galaxy.web import url_for -from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin -from galaxy.web.form_builder import CheckboxField, build_select_field +from galaxy.web.base.controller import BaseUIController +from galaxy.web.base.controller import UsesFormDefinitionsMixin +from galaxy.web.form_builder import CheckboxField +from galaxy.web.form_builder import build_select_field from galaxy.web.framework.helpers import time_ago, grids from datetime import datetime, timedelta +from galaxy.web.framework.helpers import grids from galaxy.util import hash_util log = logging.getLogger( __name__ ) @@ -104,6 +114,7 @@ use_panels=use_panels, message=message, status='error' ) ) + @web.expose def openid_process( self, trans, **kwd ): '''Handle's response from OpenID Providers''' @@ -231,6 +242,7 @@ redirect=redirect, message=message, status=status ) ) + @web.expose def openid_associate( self, trans, cntrller='user', **kwd ): '''Associates a user with an OpenID log in''' @@ -369,6 +381,7 @@ user_type_form_definition=user_type_form_definition, widgets=widgets, openids=openids ) + @web.expose @web.require_login( 'manage OpenIDs' ) def openid_disassociate( self, trans, **kwd ): @@ -849,6 +862,7 @@ else: user_type_form_definition = None return user_type_form_definition + def __get_widgets( self, trans, user_type_form_definition, user=None, **kwd ): widgets = [] if user_type_form_definition: @@ -950,6 +964,7 @@ username=user.username, message=message, status=status ) + @web.expose def edit_info( self, trans, cntrller, **kwd ): params = util.Params( kwd ) @@ -1058,6 +1073,7 @@ action='manage_user_info', cntrller=cntrller, **kwd ) ) + @web.expose def reset_password( self, trans, email=None, **kwd ): if trans.app.config.smtp_server is None: @@ -1105,6 +1121,7 @@ return trans.fill_template( '/user/reset_password.mako', message=message, status=status ) + def __validate( self, trans, params, email, password, confirm, username ): # If coming from the tool shed webapp, we'll require a public user name if trans.webapp.name == 'tool_shed': @@ -1126,6 +1143,7 @@ if user_type_fd_id in [ 'none' ]: return "Select the user's type and information" return message + @web.expose def set_default_permissions( self, trans, cntrller, **kwd ): """Sets the user's default permissions for the new histories""" @@ -1152,6 +1170,127 @@ else: # User not logged in, history group must be only public return trans.show_error_message( "You must be logged in to change your default permitted actions." ) + + @web.expose + @web.require_login() + def toolbox_filters( self, trans, cntrller, **kwd ): + """ + Sets the user's default filters for the toolbox. + Toolbox filters are specified in universe_wsgi.ini. + The user can activate them and the choice is stored in user_preferences. + """ + + def get_filter_mapping( db_filters, config_filters ): + """ + Compare the allowed filters from the universe_wsgi.ini config file with the previously saved or default filters from the database. + We need that to toogle the checkboxes for the formular in the right way. + Furthermore we extract all information associated to a filter to display them in the formular. + """ + filters = list() + for filter_name in config_filters: + if ":" in filter_name: + # Should be a submodule of filters (e.g. examples:restrict_development_tools) + (module_name, function_name) = filter_name.rsplit(":", 1) + module_name = 'galaxy.tools.filters.%s' % module_name.strip() + module = __import__( module_name, globals(), fromlist=['temp_module'] ) + function = getattr( module, function_name.strip() ) + else: + # No module found it has to be explicitly imported. + module = __import__( 'galaxy.tools.filters', globals(), fromlist=['temp_module'] ) + function = getattr( globals(), filter_name.strip() ) + + doc_string = docstring_trim( function.__doc__ ) + split = doc_string.split('\n\n') + if split: + sdesc = split[0] + else: + log.error( 'No description specified in the __doc__ string for %s.' % filter_name ) + if len(split) > 1: + description = split[1] + else: + description = '' + + if filter_name in db_filters: + filters.append( dict( filterpath=filter_name, short_desc=sdesc, desc=description, checked=True ) ) + else: + filters.append( dict( filterpath=filter_name, short_desc=sdesc, desc=description, checked=False ) ) + return filters + + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + + user_id = params.get( 'user_id', False ) + if user_id: + user = trans.sa_session.query( trans.app.model.User ).get( trans.security.decode_id( user_id ) ) + else: + user = trans.user + + if user: + saved_user_tool_filters = list() + saved_user_section_filters = list() + saved_user_label_filters = list() + + for name, value in user.preferences.items(): + if name == 'toolbox_tool_filters': + saved_user_tool_filters = listify( value, do_strip=True ) + elif name == 'toolbox_section_filters': + saved_user_section_filters = listify( value, do_strip=True ) + elif name == 'toolbox_label_filters': + saved_user_label_filters = listify( value, do_strip=True ) + + tool_filters = get_filter_mapping( saved_user_tool_filters, trans.app.config.user_tool_filters ) + section_filters = get_filter_mapping( saved_user_section_filters, trans.app.config.user_section_filters ) + label_filters = get_filter_mapping( saved_user_label_filters, trans.app.config.user_label_filters ) + + return trans.fill_template( 'user/toolbox_filters.mako', + cntrller=cntrller, + message=message, + tool_filters=tool_filters, + section_filters=section_filters, + label_filters=label_filters, + user=user, + status=status ) + else: + # User not logged in, history group must be only public + return trans.show_error_message( "You must be logged in to change private toolbox filters." ) + + @web.expose + @web.require_login( "to change the private toolbox filters" ) + def edit_toolbox_filters( self, trans, cntrller, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + user_id = params.get( 'user_id', False ) + if not user_id: + # User must be logged in to create a new address + return trans.show_error_message( "You must be logged in to change the ToolBox filters." ) + + user = trans.sa_session.query( trans.app.model.User ).get( trans.security.decode_id( user_id ) ) + + if params.get( 'edit_toolbox_filter_button', False ): + tool_filters = list() + section_filters = list() + label_filters = list() + for name, state in params.flatten(): + if state == 'on': + if name.startswith('t_'): + tool_filters.append( name[2:] ) + elif name.startswith('l_'): + label_filters.append( name[2:] ) + elif name.startswith('s_'): + section_filters.append( name[2:] ) + user.preferences['toolbox_tool_filters'] = ','.join( tool_filters ) + user.preferences['toolbox_section_filters'] = ','.join( section_filters ) + user.preferences['toolbox_label_filters'] = ','.join( label_filters ) + + trans.sa_session.add( user ) + trans.sa_session.flush() + message = 'ToolBox filters has been updated.' + kwd = dict( message=message, status='done' ) + + # Display the ToolBox filters form with the current values filled in + return self.toolbox_filters( trans, cntrller, **kwd ) + @web.expose @web.require_login( "to get most recently used tool" ) @web.json_pretty @@ -1175,6 +1314,7 @@ "description" : tool.description } return tool_info + @web.expose def manage_addresses(self, trans, **kwd): if trans.user: @@ -1196,6 +1336,7 @@ else: # User not logged in, history group must be only public return trans.show_error_message( "You must be logged in to change your default permitted actions." ) + @web.expose def new_address( self, trans, cntrller, **kwd ): params = util.Params( kwd ) @@ -1280,6 +1421,7 @@ phone=phone, message=message, status=status ) + @web.expose def edit_address( self, trans, cntrller, **kwd ): params = util.Params( kwd ) @@ -1357,6 +1499,7 @@ address_obj=address_obj, message=message, status=status ) + @web.expose def delete_address( self, trans, cntrller, address_id=None, user_id=None ): try: @@ -1377,6 +1520,7 @@ user_id=user_id, message=message, status=status ) ) + @web.expose def undelete_address( self, trans, cntrller, address_id=None, user_id=None ): try: @@ -1396,6 +1540,7 @@ user_id=user_id, message=message, status=status ) ) + @web.expose def set_user_pref_async( self, trans, pref_name, pref_value ): """ Set a user preference asynchronously. If user is not logged in, do nothing. """ diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 templates/user/index.mako --- a/templates/user/index.mako +++ b/templates/user/index.mako @@ -13,6 +13,7 @@ <li><a href="${h.url_for( controller='user', action='manage_user_info', cntrller=cntrller )}">${_('Manage your information')}</a></li><li><a href="${h.url_for( controller='user', action='set_default_permissions', cntrller=cntrller )}">${_('Change default permissions')}</a> for new histories</li><li><a href="${h.url_for( controller='user', action='api_keys', cntrller=cntrller )}">${_('Manage your API keys')}</a></li> + <li><a href="${h.url_for( controller='user', action='toolbox_filters', cntrller=cntrller )}">${_('Manage your ToolBox filters')}</a></li> %if trans.app.config.enable_openid: <li><a href="${h.url_for( controller='user', action='openid_manage', cntrller=cntrller )}">${_('Manage OpenIDs')}</a> linked to your account</li> %endif diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 templates/user/toolbox_filters.mako --- /dev/null +++ b/templates/user/toolbox_filters.mako @@ -0,0 +1,91 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, status )} +%endif +</br> +</br> + +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( controller='user', action='index', cntrller=cntrller )}">User preferences</a> + </li> +</ul> + +%if tool_filters or section_filters or label_filters: + <div class="toolForm"> + <form name="toolbox_filter" id="toolbox_filter" action="${h.url_for( controller='user', action='edit_toolbox_filters', cntrller=cntrller, user_id=trans.security.encode_id( user.id ) )}" method="post" > + % if tool_filters: + <div class="toolFormTitle">Edit ToolBox filters :: Tools</div> + <div class="toolFormBody"> + % for filter in tool_filters: + <div class="form-row"> + <div style="float: left; width: 40px; margin-right: 10px;"> + % if filter['checked']: + <input type="checkbox" name="t_${filter['filterpath']}" checked="checked"> + % else: + <input type="checkbox" name="t_${filter['filterpath']}"> + % endif + </div> + <div style="float: left; margin-right: 10px;"> + ${filter['short_desc']} + <div class="toolParamHelp" style="clear: both;">${filter['desc']}</div> + </div> + <div style="clear: both"></div> + </div> + % endfor + </div> + % endif + + % if section_filters: + <div class="toolFormTitle">Edit ToolBox filters :: Sections</div> + <div class="toolFormBody"> + % for filter in section_filters: + <div class="form-row"> + <div style="float: left; width: 40px; margin-right: 10px;"> + % if filter['checked']: + <input type="checkbox" name="s_${filter['filterpath']}" checked="checked"> + % else: + <input type="checkbox" name="s_${filter['filterpath']}"> + % endif + </div> + <div style="float: left; margin-right: 10px;"> + ${filter['short_desc']} + <div class="toolParamHelp" style="clear: both;">${filter['desc']}</div> + </div> + <div style="clear: both"></div> + </div> + % endfor + </div> + % endif + + % if label_filters: + <div class="toolFormTitle">Edit ToolBox filters :: Labels</div> + <div class="toolFormBody"> + % for filter in label_filters: + <div class="form-row"> + <div style="float: left; width: 40px; margin-right: 10px;"> + % if filter['checked']: + <input type="checkbox" name="l_${filter['filterpath']}" checked="checked"> + % else: + <input type="checkbox" name="l_${filter['filterpath']}"> + % endif + </div> + <div style="float: left; margin-right: 10px;"> + ${filter['short_desc']} + <div class="toolParamHelp" style="clear: both;">${filter['desc']}</div> + </div> + <div style="clear: both"></div> + </div> + % endfor + </div> + % endif + <div class="form-row"> + <input type="submit" name="edit_toolbox_filter_button" value="Save changes"> + </div> + </form> + </div> +%else: + ${render_msg( 'No filter available. Contact you system administrator or check your configuration file.', 'info' )} +%endif diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r fb1b676f51b224a4f3b830402d79652850dd9261 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -758,6 +758,18 @@ # if running many handlers. #cache_user_job_count = False +# ToolBox filtering +# Modules from lib/galaxy/tools/filters/ can be specified in the following lines. +# tool_* filters will be applied for all users and can not be changed by them. +# user_tool_* filters will be shown under user preferences and can be toogled +# on and off by runtime +#tool_filters +#tool_label_filters +#tool_section_filters +#user_tool_filters = examples:restrict_upload_to_admins, examples:restrict_encode +#user_tool_section_filters = examples:restrict_text +#user_tool_label_filters = examples:restrict_upload_to_admins, examples:restrict_encode + # ---- Galaxy Message Queue ------------------------------------------------- # Galaxy uses AMQ protocol to receive messages from external sources like https://bitbucket.org/galaxy/galaxy-central/commits/74100455218f/ Changeset: 74100455218f User: dannon Date: 2013-10-17 19:41:45 Summary: Fix duplicate import from bad merge Affected #: 1 file diff -r fb1b676f51b224a4f3b830402d79652850dd9261 -r 74100455218f1dbfb2ef39aceda4f88ae15e50ef lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -28,7 +28,6 @@ from galaxy.web.form_builder import build_select_field from galaxy.web.framework.helpers import time_ago, grids from datetime import datetime, timedelta -from galaxy.web.framework.helpers import grids from galaxy.util import hash_util log = logging.getLogger( __name__ ) https://bitbucket.org/galaxy/galaxy-central/commits/bdf393fb4155/ Changeset: bdf393fb4155 User: dannon Date: 2013-10-17 20:09:07 Summary: Fix indentation from merge Affected #: 1 file diff -r 74100455218f1dbfb2ef39aceda4f88ae15e50ef -r bdf393fb4155bed5f633d2daf9cd6a96ab46e7ce lib/galaxy/tools/filters/__init__.py --- a/lib/galaxy/tools/filters/__init__.py +++ b/lib/galaxy/tools/filters/__init__.py @@ -39,8 +39,8 @@ if category: self.__init_filters( category, user_filters, filters ) else: - if kwds.get( "trackster", False ): - filters[ "tool" ].append( _has_trackster_conf ) + if kwds.get( "trackster", False ): + filters[ "tool" ].append( _has_trackster_conf ) return filters https://bitbucket.org/galaxy/galaxy-central/commits/f888228f48d8/ Changeset: f888228f48d8 User: dannon Date: 2013-10-17 21:31:15 Summary: Merge w/ central Affected #: 1 file diff -r bdf393fb4155bed5f633d2daf9cd6a96ab46e7ce -r f888228f48d89ade8b8f49a4d777c573174392d1 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -293,7 +293,7 @@ full_path_to_dir = os.path.abspath( install_dir ) td_common_util.move_file( current_dir=work_dir, source=downloaded_filename, - destination_dir=full_path_to_dir ) + destination=full_path_to_dir ) if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] @@ -551,7 +551,7 @@ full_path_to_dir = os.path.abspath( install_dir ) td_common_util.move_file( current_dir=work_dir, source=downloaded_filename, - destination_dir=full_path_to_dir ) + destination=full_path_to_dir ) def log_results( command, fabric_AttributeString, file_path ): """ https://bitbucket.org/galaxy/galaxy-central/commits/acf1a0062ea8/ Changeset: acf1a0062ea8 Branch: custom-toolbox2 User: dannon Date: 2013-10-17 21:32:40 Summary: Close feature branch for toolbox filtering 2 Affected #: 0 files Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for the call to td_common_util.move_file reported by Jim Johnson - thanks Jim!
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1846eddc72a8/ Changeset: 1846eddc72a8 User: greg Date: 2013-10-17 19:46:03 Summary: Fix for the call to td_common_util.move_file reported by Jim Johnson - thanks Jim! Affected #: 1 file diff -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 -r 1846eddc72a8b4138a2835d93b5db764524c216d lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -293,7 +293,7 @@ full_path_to_dir = os.path.abspath( install_dir ) td_common_util.move_file( current_dir=work_dir, source=downloaded_filename, - destination_dir=full_path_to_dir ) + destination=full_path_to_dir ) if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] @@ -551,7 +551,7 @@ full_path_to_dir = os.path.abspath( install_dir ) td_common_util.move_file( current_dir=work_dir, source=downloaded_filename, - destination_dir=full_path_to_dir ) + destination=full_path_to_dir ) def log_results( command, fabric_AttributeString, file_path ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Graph datatypes: re-add to conf.sample returning False from all sniffers; the user must now manually change the type
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f1d77cc91367/ Changeset: f1d77cc91367 User: carlfeberhard Date: 2013-10-17 19:14:11 Summary: Graph datatypes: re-add to conf.sample returning False from all sniffers; the user must now manually change the type Affected #: 2 files diff -r 01975234fe5b2ea1058dee188b5032155a43499b -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -245,6 +245,10 @@ <datatype extension="snpmatrix" type="galaxy.datatypes.genetics:SNPMatrix" display_in_upload="true"/><datatype extension="xls" type="galaxy.datatypes.tabular:Tabular"/> +  + <datatype extension="xgmml" type="galaxy.datatypes.graph:Xgmml" display_in_upload="true"/> + <datatype extension="sif" type="galaxy.datatypes.graph:Sif" display_in_upload="true"/> + <datatype extension="rdf" type="galaxy.datatypes.graph:Rdf" display_in_upload="true"/></registration><sniffers><!-- diff -r 01975234fe5b2ea1058dee188b5032155a43499b -r f1d77cc9136794e0272f2ed9b1b33f976f3a0f10 lib/galaxy/datatypes/graph.py --- a/lib/galaxy/datatypes/graph.py +++ b/lib/galaxy/datatypes/graph.py @@ -34,14 +34,9 @@ def sniff( self, filename ): """ - Determines whether the file is XML or not, should probably actually check if it is a real xgmml file.... + Returns false and the user must manually set. """ - line = '' - with open( filename ) as handle: - line = handle.readline() - - #TODO - Is there a more robust way to do this? - return line.startswith( '<?xml ' ) + return False @staticmethod def merge( split_files, output_file ): @@ -85,19 +80,9 @@ def sniff( self, filename ): """ - Determines whether the file is SIF + Returns false and the user must manually set. """ - line = '' - with open( filename ) as infile: - correct = True - for line in infile: - if not line.strip(): - continue - tlen = len( line.split( "\t" ) ) - # may contain 1 or >= 3 columns - if tlen == 2: - correct = False - return correct + return False @staticmethod def merge( split_files, output_file ): @@ -116,6 +101,12 @@ """ file_ext = "rdf" + def sniff( self, filename ): + """ + Returns false and the user must manually set. + """ + return False + def set_peek( self, dataset, is_multi_byte=False ): """Set the peek and blurb text""" if not dataset.dataset.purged: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Allow for skipping specified repositories when setting metadata in the tool shed.
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/01975234fe5b/ Changeset: 01975234fe5b User: greg Date: 2013-10-17 17:18:05 Summary: Allow for skipping specified repositories when setting metadata in the tool shed. Affected #: 3 files diff -r 329ea7a83af4f389a7c95ee4559d88c6fec0211b -r 01975234fe5b2ea1058dee188b5032155a43499b lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -230,6 +230,7 @@ to True will restrict resetting metadata to only repositories that are writable by the user in addition to those repositories of type tool_dependency_definition. This param is ignored if the current user is not an admin user, in which case this same restriction is automatic. + :param encoded_ids_to_skip (optional): a list of encoded repository ids for repositories that should not be processed. """ def handle_repository( trans, repository, results ): repository_id = trans.security.encode_id( repository.id ) @@ -253,6 +254,7 @@ successful_count=0, unsuccessful_count=0 ) handled_repository_ids = [] + encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] ) if trans.user_is_admin(): my_writable = util.asbool( payload.get( 'my_writable', False ) ) else: @@ -260,11 +262,19 @@ query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) # First reset metadata on all repositories of type repository_dependency_definition. for repository in query: - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + encoded_id = trans.security.encode_id( repository.id ) + if encoded_id in encoded_ids_to_skip: + log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ + ( str( repository_id ), str( encoded_ids_to_skip ) ) ) + elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: results = handle_repository( trans, repository, results ) # Now reset metadata on all remaining repositories. for repository in query: - if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + encoded_id = trans.security.encode_id( repository.id ) + if encoded_id in encoded_ids_to_skip: + log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ + ( str( repository_id ), str( encoded_ids_to_skip ) ) ) + elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: results = handle_repository( trans, repository, results ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time diff -r 329ea7a83af4f389a7c95ee4559d88c6fec0211b -r 01975234fe5b2ea1058dee188b5032155a43499b lib/tool_shed/scripts/api/reset_metadata_on_repositories.py --- a/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py +++ b/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py @@ -27,18 +27,29 @@ def main( options ): api_key = options.api + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) my_writable = options.my_writable one_per_request = options.one_per_request - base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + skip_file = options.skip_file + encoded_ids_to_skip = [] + if skip_file and os.path.exists( skip_file ): + # Contents of file must be 1 encoded repository id per line. + contents = open( skip_file, 'rb' ).read() + if contents: + encoded_ids_to_skip = contents.split( '\n' ) if string_as_bool( one_per_request ): url = '%s/api/repositories/repository_ids_for_setting_metadata?key=%s&my_writable=%s' % ( base_tool_shed_url, api_key, str( my_writable ) ) repository_ids = get( url, api_key ) for repository_id in repository_ids: - data = dict( repository_id=repository_id ) - url = '%s/api/repositories/reset_metadata_on_repository' % base_tool_shed_url - submit( url, data, options.api ) + if repository_id in encoded_ids_to_skip: + print "Skipping repository with id %s because it is in skip file %s" % ( str( repository_id ), str( skip_file ) ) + else: + data = dict( repository_id=repository_id ) + url = '%s/api/repositories/reset_metadata_on_repository' % base_tool_shed_url + submit( url, data, options.api ) else: - data = dict( my_writable=my_writable ) + data = dict( encoded_ids_to_skip=encoded_ids_to_skip, + my_writable=my_writable ) url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url submit( url, data, options.api ) @@ -46,7 +57,8 @@ parser = argparse.ArgumentParser( description='Reset metadata on certain repositories in the Tool Shed via the Tool Shed API.' ) parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) parser.add_argument( "-m", "--my_writable", dest="my_writable", required=False, default='False', help="Restrict to my writable repositories" ) - parser.add_argument( "-o", "--one_per_request", dest="one_per_request", required=False, default='False', help="One repository per request" ) + parser.add_argument( "-o", "--one_per_request", dest="one_per_request", required=False, default='True', help="One repository per request" ) + parser.add_argument( "-s", "--skip_file", dest="skip_file", required=False, help="Name of local file containing encoded repository ids to skip" ) parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) options = parser.parse_args() main( options ) diff -r 329ea7a83af4f389a7c95ee4559d88c6fec0211b -r 01975234fe5b2ea1058dee188b5032155a43499b lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -821,12 +821,12 @@ clause_list = [] for repository in trans.sa_session.query( trans.model.Repository ) \ .filter( trans.model.Repository.table.c.deleted == False ): - allow_push = repository.allow_push( trans.app ) - if not order: - # We've been called from the Tool Shed API, so reset metadata on all repositories of type tool_dependency_definition. - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - elif allow_push: + # Always reset metadata on all repositories of type tool_dependency_definition. + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + else: + allow_push = repository.allow_push( trans.app ) + if allow_push: # Include all repositories that are writable by the current user. allow_push_usernames = allow_push.split( ',' ) if username in allow_push_usernames: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bug fix related to return code handling in TaskWrapper.
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/329ea7a83af4/ Changeset: 329ea7a83af4 User: jmchilton Date: 2013-10-17 12:30:13 Summary: Bug fix related to return code handling in TaskWrapper. Thanks to Peter Cock for finding this. Affected #: 2 files diff -r 8e001dc9675c105d83d83f6d5a756c11759e8bb2 -r 329ea7a83af4f389a7c95ee4559d88c6fec0211b lib/galaxy/jobs/command_factory.py --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -55,18 +55,20 @@ # Append metadata setting commands, we don't want to overwrite metadata # that was copied over in init_meta(), as per established behavior if include_metadata and job_wrapper.requires_setting_metadata: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True - commands += "; cd %s; " % abspath( getcwd() ) - commands += job_wrapper.setup_external_metadata( + metadata_command = job_wrapper.setup_external_metadata( exec_dir=abspath( getcwd() ), tmp_dir=job_wrapper.working_directory, dataset_files_path=job.app.model.Dataset.file_path, output_fnames=job_wrapper.get_output_fnames(), set_extension=False, kwds={ 'overwrite' : False } - ) + ) or '' + metadata_command = metadata_command.strip() + if metadata_command: + if not captured_return_code: + commands += capture_return_code_command + captured_return_code = True + commands += "; cd %s; %s" % (abspath( getcwd() ), metadata_command) if captured_return_code: commands += '; sh -c "exit $return_code"' diff -r 8e001dc9675c105d83d83f6d5a756c11759e8bb2 -r 329ea7a83af4f389a7c95ee4559d88c6fec0211b test/unit/test_command_factory.py --- a/test/unit/test_command_factory.py +++ b/test/unit/test_command_factory.py @@ -45,6 +45,18 @@ expected_command = '%s; return_code=$?; cd %s; %s; sh -c "exit $return_code"' % (MOCK_COMMAND_LINE, getcwd(), metadata_line) self.__assert_command_is( expected_command ) + def test_empty_metadata(self): + """ + As produced by TaskWrapper. + """ + self.include_metadata = True + self.include_work_dir_outputs = False + self.job_wrapper.metadata_line = ' ' + # Empty metadata command do not touch command line. + expected_command = '%s' % (MOCK_COMMAND_LINE) + self.__assert_command_is( expected_command ) + + def __assert_command_is(self, expected_command): command = self.__command() self.assertEqual(command, expected_command) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 24 new changesets
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

24 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/be16cffdcb68/ Changeset: be16cffdcb68 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Convert James Taylor's dependency testing file lib/galaxy/tools/deps/tests.py into a formal unit test. Affected #: 2 files diff -r ebd092733e552ef6a846d55b69d22dc779dfcc5f -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 lib/galaxy/tools/deps/tests.py --- a/lib/galaxy/tools/deps/tests.py +++ /dev/null @@ -1,32 +0,0 @@ -import tempfile -import os.path -from os import makedirs, mkdir -import galaxy.tools.deps - -def touch( fname, data=None ): - f = open( fname, 'w' ) - if data: - f.write( data ) - f.close() - -def test(): - - # Setup directories - base_path = tempfile.mkdtemp() - # mkdir( base_path ) - for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: - if sub == "bin": - p = os.path.join( base_path, name, version, "bin" ) - else: - p = os.path.join( base_path, name, version ) - try: - makedirs( p ) - except: - pass - if sub == "env.sh": - touch( os.path.join( p, "env.sh" ) ) - - dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) - - print dm.find_dep( "dep1", "1.0" ) - print dm.find_dep( "dep1", "2.0" ) diff -r ebd092733e552ef6a846d55b69d22dc779dfcc5f -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 test/unit/test_tool_deps.py --- /dev/null +++ b/test/unit/test_tool_deps.py @@ -0,0 +1,40 @@ +import tempfile +import os.path +from os import makedirs +import galaxy.tools.deps + + +def touch( fname, data=None ): + f = open( fname, 'w' ) + if data: + f.write( data ) + f.close() + + +def test_tool_dependencies(): + + # Setup directories + base_path = tempfile.mkdtemp() + # mkdir( base_path ) + for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: + if sub == "bin": + p = os.path.join( base_path, name, version, "bin" ) + else: + p = os.path.join( base_path, name, version ) + try: + makedirs( p ) + except: + pass + if sub == "env.sh": + touch( os.path.join( p, "env.sh" ) ) + + dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) + + d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) + assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) + assert d1_version == "1.0" + d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) + assert d2_script == None + assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) + assert d2_version == "2.0" https://bitbucket.org/galaxy/galaxy-central/commits/e276f3fc9a74/ Changeset: e276f3fc9a74 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Add tests for addditional existing behavior for tool dependencies. Namely test the fallen: - Test default package will not be fallen back upon when exactly specified version is used. (Not my favorite behavior. -John) - Test tool shed package and set_environment requirement types (standard use). - Test tool shed installed tools will fallback on traditional Galaxy dependencies if a requirement is unmatched in installed dependencies. - Test tool shed dependency takes prescndence over traditional Galaxy dependency when both are available. Other small clean ups of test/unit/test_tool_deps.py. Affected #: 1 file diff -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 -r e276f3fc9a7426923d968e7e56da1e3b26292056 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,40 +1,140 @@ import tempfile import os.path -from os import makedirs -import galaxy.tools.deps - - -def touch( fname, data=None ): - f = open( fname, 'w' ) - if data: - f.write( data ) - f.close() +from os import makedirs, symlink +from shutil import rmtree +from galaxy.tools.deps import DependencyManager +from galaxy.util.bunch import Bunch +from contextlib import contextmanager def test_tool_dependencies(): + # Setup directories + with __test_base_path() as base_path: + for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: + if sub == "bin": + p = os.path.join( base_path, name, version, "bin" ) + else: + p = os.path.join( base_path, name, version ) + try: + makedirs( p ) + except: + pass + if sub == "env.sh": + __touch( os.path.join( p, "env.sh" ) ) - # Setup directories + dm = DependencyManager( [base_path] ) + d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) + assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) + assert d1_version == "1.0" + d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) + assert d2_script == None + assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) + assert d2_version == "2.0" + + ## Test default versions + symlink( os.path.join( base_path, 'dep1', '2.0'), os.path.join( base_path, 'dep1', 'default' ) ) + default_script, default_path, default_version = dm.find_dep( "dep1", None ) + assert default_version == "2.0" + + ## Test default will not be fallen back upon by default + default_script, default_path, default_version = dm.find_dep( "dep1", "2.1" ) + assert default_script == None + assert default_version == None + + +TEST_REPO_USER = "devteam" +TEST_REPO_NAME = "bwa" +TEST_REPO_CHANGESET = "12abcd41223da" +TEST_VERSION = "0.5.9" + + +def test_toolshed_set_enviornment_requiremetns(): + with __test_base_path() as base_path: + test_repo = __build_test_repo('set_environment') + dm = DependencyManager( [base_path] ) + env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) + os.makedirs(env_settings_dir) + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) + assert d1_version == None + assert d1_script == os.path.join(env_settings_dir, "env.sh"), d1_script + + +def test_toolshed_package_requirements(): + with __test_base_path() as base_path: + test_repo = __build_test_repo('package', version=TEST_VERSION) + dm = DependencyManager( [base_path] ) + package_dir = __build_ts_test_package(base_path) + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert d1_version == TEST_VERSION, d1_version + assert d1_script == os.path.join(package_dir, "env.sh"), d1_script + + +def test_toolshed_tools_fallback_on_manual_dependencies(): + with __test_base_path() as base_path: + dm = DependencyManager( [base_path] ) + test_repo = __build_test_repo('package', version=TEST_VERSION) + env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") + d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) + assert d1_version == "1.0" + assert d1_script == env_path + + +def test_toolshed_greater_precendence(): + with __test_base_path() as base_path: + dm = DependencyManager( [base_path] ) + test_repo = __build_test_repo('package', version=TEST_VERSION) + ts_package_dir = __build_ts_test_package(base_path) + gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) + ts_env_path = os.path.join(ts_package_dir, "env.sh") + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert d1_script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert d1_script == ts_env_path + + +def __build_ts_test_package(base_path, script_contents=''): + package_dir = os.path.join(base_path, TEST_REPO_NAME, TEST_VERSION, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) + __touch(os.path.join(package_dir, 'env.sh'), script_contents) + return package_dir + + +def __setup_galaxy_package_dep(base_path, name, version, contents=""): + dep_directory = os.path.join( base_path, name, version ) + env_path = os.path.join( dep_directory, "env.sh" ) + __touch( env_path, contents ) + return env_path + + +def __touch( fname, data=None ): + dirname = os.path.dirname( fname ) + if not os.path.exists( dirname ): + makedirs( dirname ) + f = open( fname, 'w' ) + try: + if data: + f.write( data ) + finally: + f.close() + + +def __build_test_repo(type, version=None): + return Bunch( + owner=TEST_REPO_USER, + name=TEST_REPO_NAME, + type=type, + version=version, + tool_shed_repository=Bunch( + owner=TEST_REPO_USER, + name=TEST_REPO_NAME, + installed_changeset_revision=TEST_REPO_CHANGESET + ) + ) + + +@contextmanager +def __test_base_path(): base_path = tempfile.mkdtemp() - # mkdir( base_path ) - for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: - if sub == "bin": - p = os.path.join( base_path, name, version, "bin" ) - else: - p = os.path.join( base_path, name, version ) - try: - makedirs( p ) - except: - pass - if sub == "env.sh": - touch( os.path.join( p, "env.sh" ) ) - - dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) - - d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) - assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) - assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) - assert d1_version == "1.0" - d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) - assert d2_script == None - assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) - assert d2_version == "2.0" + try: + yield base_path + finally: + rmtree(base_path) https://bitbucket.org/galaxy/galaxy-central/commits/b16ab0d407a0/ Changeset: b16ab0d407a0 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: PEP8 fixes for lib/galaxy/tools/deps/__init__.py Affected #: 1 file diff -r e276f3fc9a7426923d968e7e56da1e3b26292056 -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,6 +7,7 @@ import logging log = logging.getLogger( __name__ ) + class DependencyManager( object ): """ A DependencyManager attempts to resolve named and versioned dependencies by searching for them under a list of directories. Directories should be @@ -28,6 +29,7 @@ if not os.path.isdir( base_path ): log.warn( "Path '%s' is not directory, ignoring", base_path ) self.base_paths.append( os.path.abspath( base_path ) ) + def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a @@ -37,6 +39,7 @@ return self._find_dep_default( name, type=type, installed_tool_dependencies=installed_tool_dependencies ) else: return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) for base_path in self.base_paths: @@ -51,6 +54,7 @@ return None, path, version else: return None, None, None + def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=None ) @@ -68,20 +72,22 @@ script = os.path.join( real_path, 'env.sh' ) if os.path.exists( script ): return script, real_path, real_version - elif os.path.exists( os.path.join( real_path, 'bin' ) ): + elif os.path.exists( real_bin ): return None, real_path, real_version else: return None, None, None + def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: for installed_tool_dependency in installed_tool_dependencies: if version: - if installed_tool_dependency.name==name and installed_tool_dependency.type==type and installed_tool_dependency.version==version: + if installed_tool_dependency.name == name and installed_tool_dependency.type == type and installed_tool_dependency.version == version: return installed_tool_dependency else: - if installed_tool_dependency.name==name and installed_tool_dependency.type==type: + if installed_tool_dependency.name == name and installed_tool_dependency.type == type: return installed_tool_dependency return None + def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): tool_shed_repository = installed_tool_dependency.tool_shed_repository return os.path.join( base_path, @@ -90,6 +96,7 @@ tool_shed_repository.owner, tool_shed_repository.name, tool_shed_repository.installed_changeset_revision ) + def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository for base_path in self.base_paths: https://bitbucket.org/galaxy/galaxy-central/commits/630cfe0030ab/ Changeset: 630cfe0030ab User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Drop multiple tool_dependency_dir implementation for now. Will reimplement once I have added more granular, plugin style tool dependency resolvers. They will benefit from there being just one default_base_path as implemented here, but allowing it to be overridden on a per resolver basis. Affected #: 3 files diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -696,7 +696,7 @@ def init_dependency_manager( self ): if self.app.config.use_tool_dependencies: - self.dependency_manager = DependencyManager( [ self.app.config.tool_dependency_dir ] ) + self.dependency_manager = DependencyManager( self.app.config.tool_dependency_dir ) else: self.dependency_manager = None diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -17,18 +17,16 @@ and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, base_paths=[] ): + def __init__( self, default_base_path ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. """ - self.base_paths = [] - for base_path in base_paths: - if not os.path.exists( base_path ): - log.warn( "Path '%s' does not exist, ignoring", base_path ) - if not os.path.isdir( base_path ): - log.warn( "Path '%s' is not directory, ignoring", base_path ) - self.base_paths.append( os.path.abspath( base_path ) ) + if not os.path.exists( default_base_path ): + log.warn( "Path '%s' does not exist, ignoring", default_base_path ) + if not os.path.isdir( default_base_path ): + log.warn( "Path '%s' is not directory, ignoring", default_base_path ) + self.default_base_path = os.path.abspath( default_base_path ) def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): """ @@ -42,18 +40,17 @@ def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) - for base_path in self.base_paths: - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - else: - path = os.path.join( base_path, name, version ) - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version + base_path = self.default_base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) else: - return None, None, None + path = os.path.join( base_path, name, version ) + script = os.path.join( path, 'env.sh' ) + if os.path.exists( script ): + return script, path, version + elif os.path.exists( os.path.join( path, 'bin' ) ): + return None, path, version + return None, None, None def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: @@ -63,19 +60,18 @@ if script and path: # Environment settings do not use versions. return script, path, None - for base_path in self.base_paths: - path = os.path.join( base_path, name, 'default' ) - if os.path.islink( path ): - real_path = os.path.realpath( path ) - real_bin = os.path.join( real_path, 'bin' ) - real_version = os.path.basename( real_path ) - script = os.path.join( real_path, 'env.sh' ) - if os.path.exists( script ): - return script, real_path, real_version - elif os.path.exists( real_bin ): - return None, real_path, real_version - else: - return None, None, None + base_path = self.default_base_path + path = os.path.join( base_path, name, 'default' ) + if os.path.islink( path ): + real_path = os.path.realpath( path ) + real_bin = os.path.join( real_path, 'bin' ) + real_version = os.path.basename( real_path ) + script = os.path.join( real_path, 'env.sh' ) + if os.path.exists( script ): + return script, real_path, real_version + elif os.path.exists( real_bin ): + return None, real_path, real_version + return None, None, None def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: @@ -99,14 +95,14 @@ def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository - for base_path in self.base_paths: - path = os.path.abspath( os.path.join( base_path, - 'environment_settings', - name, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) ) - if os.path.exists( path ): - script = os.path.join( path, 'env.sh' ) - return script, path, None + base_path = self.default_base_path + path = os.path.abspath( os.path.join( base_path, + 'environment_settings', + name, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) ) + if os.path.exists( path ): + script = os.path.join( path, 'env.sh' ) + return script, path, None return None, None, None diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -9,6 +9,7 @@ def test_tool_dependencies(): # Setup directories + with __test_base_path() as base_path: for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: if sub == "bin": @@ -22,7 +23,7 @@ if sub == "env.sh": __touch( os.path.join( p, "env.sh" ) ) - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) @@ -52,7 +53,7 @@ def test_toolshed_set_enviornment_requiremetns(): with __test_base_path() as base_path: test_repo = __build_test_repo('set_environment') - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) os.makedirs(env_settings_dir) d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) @@ -63,7 +64,7 @@ def test_toolshed_package_requirements(): with __test_base_path() as base_path: test_repo = __build_test_repo('package', version=TEST_VERSION) - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) package_dir = __build_ts_test_package(base_path) d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) assert d1_version == TEST_VERSION, d1_version @@ -72,7 +73,7 @@ def test_toolshed_tools_fallback_on_manual_dependencies(): with __test_base_path() as base_path: - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) @@ -82,7 +83,7 @@ def test_toolshed_greater_precendence(): with __test_base_path() as base_path: - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) ts_package_dir = __build_ts_test_package(base_path) gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) https://bitbucket.org/galaxy/galaxy-central/commits/a54a058fa777/ Changeset: a54a058fa777 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Small clean ups. Affected #: 1 file diff -r 630cfe0030abf40e46c9839254278531d657e554 -r a54a058fa77737cd1f9d51393cb46f5bf8dda95f lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,15 +7,19 @@ import logging log = logging.getLogger( __name__ ) +INDETERMINATE_DEPENDENCY = (None, None, None) + class DependencyManager( object ): """ - A DependencyManager attempts to resolve named and versioned dependencies by searching for them under a list of directories. Directories should be + A DependencyManager attempts to resolve named and versioned dependencies by + searching for them under a list of directories. Directories should be of the form: $BASE/name/version/... - and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. + and should each contain a file 'env.sh' which can be sourced to make the + dependency available in the current shell environment. """ def __init__( self, default_base_path ): """ @@ -50,7 +54,7 @@ return script, path, version elif os.path.exists( os.path.join( path, 'bin' ) ): return None, path, version - return None, None, None + return INDETERMINATE_DEPENDENCY def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: @@ -71,16 +75,17 @@ return script, real_path, real_version elif os.path.exists( real_bin ): return None, real_path, real_version - return None, None, None + return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: for installed_tool_dependency in installed_tool_dependencies: + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type if version: - if installed_tool_dependency.name == name and installed_tool_dependency.type == type and installed_tool_dependency.version == version: + if name_and_type_equal and installed_tool_dependency.version == version: return installed_tool_dependency else: - if installed_tool_dependency.name == name and installed_tool_dependency.type == type: + if name_and_type_equal: return installed_tool_dependency return None @@ -105,4 +110,4 @@ if os.path.exists( path ): script = os.path.join( path, 'env.sh' ) return script, path, None - return None, None, None + return INDETERMINATE_DEPENDENCY https://bitbucket.org/galaxy/galaxy-central/commits/8ca4fdce3052/ Changeset: 8ca4fdce3052 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Begin rearranging arguments so all ones not of interest to every potential dependency resolution plugin can be kwds Affected #: 1 file diff -r a54a058fa77737cd1f9d51393cb46f5bf8dda95f -r 8ca4fdce30526d58737d2b0d21af15cb750ddffc lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -43,7 +43,7 @@ return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): - installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, installed_tool_dependencies=installed_tool_dependencies ) base_path = self.default_base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) @@ -58,7 +58,7 @@ def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: - installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=None ) + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, installed_tool_dependencies=installed_tool_dependencies ) if installed_tool_dependency: script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) if script and path: @@ -77,16 +77,15 @@ return None, real_path, real_version return INDETERMINATE_DEPENDENCY - def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): - if installed_tool_dependencies: - for installed_tool_dependency in installed_tool_dependencies: - name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type - if version: - if name_and_type_equal and installed_tool_dependency.version == version: - return installed_tool_dependency - else: - if name_and_type_equal: - return installed_tool_dependency + def _get_installed_dependency( self, name, type, version=None, **kwds ): + for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type + if version: + if name_and_type_equal and installed_tool_dependency.version == version: + return installed_tool_dependency + else: + if name_and_type_equal: + return installed_tool_dependency return None def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): https://bitbucket.org/galaxy/galaxy-central/commits/22f3aca8ad9c/ Changeset: 22f3aca8ad9c User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Finish refactoring toward generic keyword arguments. Affected #: 1 file diff -r 8ca4fdce30526d58737d2b0d21af15cb750ddffc -r 22f3aca8ad9cb9a0e606a1c69cb4c211fe5cc133 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -32,18 +32,18 @@ log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): + def find_dep( self, name, version=None, type='package', **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ if version is None: - return self._find_dep_default( name, type=type, installed_tool_dependencies=installed_tool_dependencies ) + return self._find_dep_default( name, type=type, **kwds ) else: - return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) + return self._find_dep_versioned( name, version, type=type, **kwds ) - def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) base_path = self.default_base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) @@ -56,9 +56,9 @@ return None, path, version return INDETERMINATE_DEPENDENCY - def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): - if type == 'set_environment' and installed_tool_dependencies: - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) if installed_tool_dependency: script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) if script and path: https://bitbucket.org/galaxy/galaxy-central/commits/dd2b36c4fc66/ Changeset: dd2b36c4fc66 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Introduce the concept of a dependency resolver, DependencyManager will check each in turn. Affected #: 1 file diff -r 22f3aca8ad9cb9a0e606a1c69cb4c211fe5cc133 -r dd2b36c4fc6682e76cceef10de2ae3154cc107c6 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -31,8 +31,29 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) + self.dependency_resolvers = [ GalaxyPackageDependencyResolver(self) ] + def find_dep( self, name, version=None, type='package', **kwds ): + for resolver in self.dependency_resolvers: + dependency = resolver.resolve( name, version, type, **kwds ) + if dependency != INDETERMINATE_DEPENDENCY: + return dependency + return INDETERMINATE_DEPENDENCY + + +class DependencyResolver(object): + + def resolve( self, name, version, type, **kwds ): + raise NotImplementedError() + + +class GalaxyPackageDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager): + self.default_base_path = dependency_manager.default_base_path + + def resolve( self, name, version, type, **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version https://bitbucket.org/galaxy/galaxy-central/commits/ae5dbc930415/ Changeset: ae5dbc930415 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Refactor common code out of _find_dep_versioned and _find_dep_default. Affected #: 1 file diff -r dd2b36c4fc6682e76cceef10de2ae3154cc107c6 -r ae5dbc9304152e034ed5565f5243ab6b5c0cbe77 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -70,12 +70,7 @@ path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) else: path = os.path.join( base_path, name, version ) - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version - return INDETERMINATE_DEPENDENCY + return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): @@ -89,13 +84,17 @@ path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): real_path = os.path.realpath( path ) - real_bin = os.path.join( real_path, 'bin' ) real_version = os.path.basename( real_path ) - script = os.path.join( real_path, 'env.sh' ) - if os.path.exists( script ): - return script, real_path, real_version - elif os.path.exists( real_bin ): - return None, real_path, real_version + return self._galaxy_package_dep(real_path, real_version) + else: + return INDETERMINATE_DEPENDENCY + + def _galaxy_package_dep( self, path, version ): + script = os.path.join( path, 'env.sh' ) + if os.path.exists( script ): + return script, path, version + elif os.path.exists( os.path.join( path, 'bin' ) ): + return None, path, version return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, name, type, version=None, **kwds ): https://bitbucket.org/galaxy/galaxy-central/commits/8928cf0be809/ Changeset: 8928cf0be809 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Split package resolvers into Tool Shed and traditional Galaxy versions. Affected #: 1 file diff -r ae5dbc9304152e034ed5565f5243ab6b5c0cbe77 -r 8928cf0be80994703689e89c96e2dbfa20a47319 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -31,7 +31,10 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - self.dependency_resolvers = [ GalaxyPackageDependencyResolver(self) ] + self.dependency_resolvers = [ + ToolShedPackageDependencyResolver(self), + GalaxyPackageDependencyResolver(self), + ] def find_dep( self, name, version=None, type='package', **kwds ): @@ -64,22 +67,11 @@ return self._find_dep_versioned( name, version, type=type, **kwds ) def _find_dep_versioned( self, name, version, type='package', **kwds ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) base_path = self.default_base_path - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - else: - path = os.path.join( base_path, name, version ) + path = os.path.join( base_path, name, version ) return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): - if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) - if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: - # Environment settings do not use versions. - return script, path, None base_path = self.default_base_path path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): @@ -97,6 +89,31 @@ return None, path, version return INDETERMINATE_DEPENDENCY + +class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + + def __init__(self, dependency_manager): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) + base_path = self.default_base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) + return self._galaxy_package_dep(path, version) + else: + return INDETERMINATE_DEPENDENCY + + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) + if installed_tool_dependency: + script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if script and path: + # Environment settings do not use versions. + return script, path, None + return INDETERMINATE_DEPENDENCY + def _get_installed_dependency( self, name, type, version=None, **kwds ): for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type https://bitbucket.org/galaxy/galaxy-central/commits/0762bd90e85b/ Changeset: 0762bd90e85b User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Implement XML configuration of tool dependency resolver. This is largely modelled off of @natefoo's job_conf.xml. This is a minimally configurable XML layout so far, but will be dding more options. Already can decide between whether to load tool shed installed packages first (default) or manually conifgured ones. The advantages of loading manually configured ones first is outlined here - http://dev.list.galaxyproject.org/Test-Toolshed-Biopython-package-dependenc…. Affected #: 4 files diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -114,6 +114,7 @@ self.collect_outputs_from = [ x.strip() for x in kwargs.get( 'collect_outputs_from', 'new_file_path,job_working_directory' ).lower().split(',') ] self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root ) self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root ) + self.dependency_resolvers_config_file = resolve_path( kwargs.get( 'dependency_resolvers_config_file', 'dependency_resolvers_conf.xml' ), self.root ) self.job_config_file = resolve_path( kwargs.get( 'job_config_file', 'job_conf.xml' ), self.root ) self.local_job_queue_workers = int( kwargs.get( "local_job_queue_workers", "5" ) ) self.cluster_job_queue_workers = int( kwargs.get( "cluster_job_queue_workers", "3" ) ) diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -696,7 +696,11 @@ def init_dependency_manager( self ): if self.app.config.use_tool_dependencies: - self.dependency_manager = DependencyManager( self.app.config.tool_dependency_dir ) + dependency_manager_kwds = { + 'default_base_path': self.app.config.tool_dependency_dir, + 'conf_file': self.app.config.dependency_resolvers_config_file, + } + self.dependency_manager = DependencyManager( **dependency_manager_kwds ) else: self.dependency_manager = None diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,6 +7,8 @@ import logging log = logging.getLogger( __name__ ) +from galaxy.util import parse_xml + INDETERMINATE_DEPENDENCY = (None, None, None) @@ -21,7 +23,7 @@ and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, default_base_path ): + def __init__( self, default_base_path, conf_file=None ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -31,10 +33,7 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - self.dependency_resolvers = [ - ToolShedPackageDependencyResolver(self), - GalaxyPackageDependencyResolver(self), - ] + self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) def find_dep( self, name, version=None, type='package', **kwds ): @@ -44,6 +43,32 @@ return dependency return INDETERMINATE_DEPENDENCY + def __build_dependency_resolvers( self, conf_file ): + if not conf_file or not os.path.exists( conf_file ): + return self.__default_dependency_resolvers() + tree = parse_xml( conf_file ) + return self.__parse_resolver_conf_xml( tree ) + + def __default_dependency_resolvers( self ): + return [ + ToolShedPackageDependencyResolver(self), + GalaxyPackageDependencyResolver(self), + ] + + def __parse_resolver_conf_xml(self, tree): + """ + + :param tree: Object representing the root ``<dependency_resolvers>`` object in the file. + :type tree: ``xml.etree.ElementTree.Element`` + """ + resolvers = [] + resolvers_element = tree.getroot() + for resolver_element in resolvers_element.getchildren(): + resolver_type = resolver_element.tag + resolver = RESOLVER_CLASSES[resolver_type](self) + resolvers.append(resolver) + return resolvers + class DependencyResolver(object): @@ -147,3 +172,9 @@ script = os.path.join( path, 'env.sh' ) return script, path, None return INDETERMINATE_DEPENDENCY + + +RESOLVER_CLASSES = { + 'tool_shed_package': ToolShedPackageDependencyResolver, + 'galaxy_package': GalaxyPackageDependencyResolver, +} diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -139,3 +139,32 @@ yield base_path finally: rmtree(base_path) + + +def test_parse(): + with __parse_resolvers('''<dependency_resolvers> + <tool_shed_package /> + <galaxy_package /> +</dependency_resolvers> +''') as dependency_resolvers: + assert 'ToolShed' in dependency_resolvers[0].__class__.__name__ + assert 'Galaxy' in dependency_resolvers[1].__class__.__name__ + + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> +</dependency_resolvers> +''') as dependency_resolvers: + assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ + assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ + + +@contextmanager +def __parse_resolvers(xml_content): + with __test_base_path() as base_path: + f = tempfile.NamedTemporaryFile() + f.write(xml_content) + f.flush() + dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) + yield dm.dependency_resolvers + https://bitbucket.org/galaxy/galaxy-central/commits/4711ced22745/ Changeset: 4711ced22745 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Add ability to pass arguments to tool resolvers. Use new functionlity to allow a much cleaner, more general solution to the problem outlined in Pull Request 220. Allow creation of a Galaxy package dependency resolver that always falls back to 'default' dependency. I think this should be the default last-ditch behavior if a specified version of a tag is not found, but I understand the desire for it not to be, hopefully making it optional in this fashion is an acceptable middle ground. The original problem is essentially, tool shed installs require an exact version of package requirements to be used, but this very encumbering in rapidly evolving and/or non-toolshed use cases (e.g. my Galaxy-P production and development servers). It is tedius to keep different code bases, environments in such cases. Affected #: 2 files diff -r 0762bd90e85b94e643458c22670de8455dc97f71 -r 4711ced227459680997ffe448ea57abd65349fb2 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,7 +7,7 @@ import logging log = logging.getLogger( __name__ ) -from galaxy.util import parse_xml +from galaxy.util import parse_xml, string_as_bool INDETERMINATE_DEPENDENCY = (None, None, None) @@ -65,7 +65,8 @@ resolvers_element = tree.getroot() for resolver_element in resolvers_element.getchildren(): resolver_type = resolver_element.tag - resolver = RESOLVER_CLASSES[resolver_type](self) + resolver_kwds = dict(resolver_element.items()) + resolver = RESOLVER_CLASSES[resolver_type](self, **resolver_kwds) resolvers.append(resolver) return resolvers @@ -78,15 +79,22 @@ class GalaxyPackageDependencyResolver(DependencyResolver): - def __init__(self, dependency_manager): + def __init__(self, dependency_manager, **kwds): self.default_base_path = dependency_manager.default_base_path + ## Galaxy tool shed requires explicit versions on XML elements, + ## this in inconvient for testing or Galaxy instances not utilizing + ## the tool shed so allow a fallback version of the Galaxy package + ## resolver that will just grab 'default' version of exact version + ## unavailable. + self.versionless = string_as_bool(kwds.get('versionless', "false")) + def resolve( self, name, version, type, **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ - if version is None: + if version is None or self.versionless: return self._find_dep_default( name, type=type, **kwds ) else: return self._find_dep_versioned( name, version, type=type, **kwds ) @@ -117,8 +125,8 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): - def __init__(self, dependency_manager): - super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager) + def __init__(self, dependency_manager, **kwds): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) def _find_dep_versioned( self, name, version, type='package', **kwds ): installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) diff -r 0762bd90e85b94e643458c22670de8455dc97f71 -r 4711ced227459680997ffe448ea57abd65349fb2 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -158,6 +158,15 @@ assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> + <galaxy_package versionless="true" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert not dependency_resolvers[0].versionless + assert dependency_resolvers[2].versionless + @contextmanager def __parse_resolvers(xml_content): https://bitbucket.org/galaxy/galaxy-central/commits/73c22fa3ae10/ Changeset: 73c22fa3ae10 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Allow overridding the base_path on 'galaxy_package' tool dependency resolvers. There was some code for doing this earlier but it wasn't configurable in anyway. This version looks like it should work and is cleaner. Affected #: 2 files diff -r 4711ced227459680997ffe448ea57abd65349fb2 -r 73c22fa3ae109e14bd70128c6e00174beae45629 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -80,14 +80,13 @@ class GalaxyPackageDependencyResolver(DependencyResolver): def __init__(self, dependency_manager, **kwds): - self.default_base_path = dependency_manager.default_base_path - ## Galaxy tool shed requires explicit versions on XML elements, ## this in inconvient for testing or Galaxy instances not utilizing ## the tool shed so allow a fallback version of the Galaxy package ## resolver that will just grab 'default' version of exact version ## unavailable. self.versionless = string_as_bool(kwds.get('versionless', "false")) + self.base_path = kwds.get('base_path', dependency_manager.default_base_path) def resolve( self, name, version, type, **kwds ): """ @@ -100,12 +99,12 @@ return self._find_dep_versioned( name, version, type=type, **kwds ) def _find_dep_versioned( self, name, version, type='package', **kwds ): - base_path = self.default_base_path + base_path = self.base_path path = os.path.join( base_path, name, version ) return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): - base_path = self.default_base_path + base_path = self.base_path path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): real_path = os.path.realpath( path ) @@ -130,7 +129,7 @@ def _find_dep_versioned( self, name, version, type='package', **kwds ): installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) - base_path = self.default_base_path + base_path = self.base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) return self._galaxy_package_dep(path, version) @@ -169,7 +168,7 @@ def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository - base_path = self.default_base_path + base_path = self.base_path path = os.path.abspath( os.path.join( base_path, 'environment_settings', name, diff -r 4711ced227459680997ffe448ea57abd65349fb2 -r 73c22fa3ae109e14bd70128c6e00174beae45629 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -167,6 +167,19 @@ assert not dependency_resolvers[0].versionless assert dependency_resolvers[2].versionless + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> + <galaxy_package base_path="/opt/galaxy/legacy/"/> +</dependency_resolvers> +''') as dependency_resolvers: + # Unspecified base_paths are both default_base_paths + assert dependency_resolvers[0].base_path == dependency_resolvers[1].base_path + # Can specify custom base path... + assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy/" + # ... that is different from the default. + assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path + @contextmanager def __parse_resolvers(xml_content): https://bitbucket.org/galaxy/galaxy-central/commits/36d6cca02ff2/ Changeset: 36d6cca02ff2 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Modify the tool resolver syntax slightly. <galaxy_packages /> looks better than <galaxy_package /> since it is a whole class of possible dependencies the corresponding resolver could resolve. Affected #: 2 files diff -r 73c22fa3ae109e14bd70128c6e00174beae45629 -r 36d6cca02ff256889ea3a65b256b098431d35192 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -182,6 +182,6 @@ RESOLVER_CLASSES = { - 'tool_shed_package': ToolShedPackageDependencyResolver, - 'galaxy_package': GalaxyPackageDependencyResolver, + 'tool_shed_packages': ToolShedPackageDependencyResolver, + 'galaxy_packages': GalaxyPackageDependencyResolver, } diff -r 73c22fa3ae109e14bd70128c6e00174beae45629 -r 36d6cca02ff256889ea3a65b256b098431d35192 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -143,34 +143,34 @@ def test_parse(): with __parse_resolvers('''<dependency_resolvers> - <tool_shed_package /> - <galaxy_package /> + <tool_shed_packages /> + <galaxy_packages /></dependency_resolvers> ''') as dependency_resolvers: assert 'ToolShed' in dependency_resolvers[0].__class__.__name__ assert 'Galaxy' in dependency_resolvers[1].__class__.__name__ with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> + <galaxy_packages /> + <tool_shed_packages /></dependency_resolvers> ''') as dependency_resolvers: assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> - <galaxy_package versionless="true" /> + <galaxy_packages /> + <tool_shed_packages /> + <galaxy_packages versionless="true" /></dependency_resolvers> ''') as dependency_resolvers: assert not dependency_resolvers[0].versionless assert dependency_resolvers[2].versionless with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> - <galaxy_package base_path="/opt/galaxy/legacy/"/> + <galaxy_packages /> + <tool_shed_packages /> + <galaxy_packages base_path="/opt/galaxy/legacy/"/></dependency_resolvers> ''') as dependency_resolvers: # Unspecified base_paths are both default_base_paths https://bitbucket.org/galaxy/galaxy-central/commits/b7380008b339/ Changeset: b7380008b339 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Refactor actual tool resolvers out into the own module. Cleans up lib/galaxy/tools/deps/__init__.py significantly and makes inter-dependencies more clear. Affected #: 4 files diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,9 +7,10 @@ import logging log = logging.getLogger( __name__ ) -from galaxy.util import parse_xml, string_as_bool - -INDETERMINATE_DEPENDENCY = (None, None, None) +from galaxy.util import parse_xml +from .resolvers import INDETERMINATE_DEPENDENCY +from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver +from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver class DependencyManager( object ): @@ -70,117 +71,6 @@ resolvers.append(resolver) return resolvers - -class DependencyResolver(object): - - def resolve( self, name, version, type, **kwds ): - raise NotImplementedError() - - -class GalaxyPackageDependencyResolver(DependencyResolver): - - def __init__(self, dependency_manager, **kwds): - ## Galaxy tool shed requires explicit versions on XML elements, - ## this in inconvient for testing or Galaxy instances not utilizing - ## the tool shed so allow a fallback version of the Galaxy package - ## resolver that will just grab 'default' version of exact version - ## unavailable. - self.versionless = string_as_bool(kwds.get('versionless', "false")) - self.base_path = kwds.get('base_path', dependency_manager.default_base_path) - - def resolve( self, name, version, type, **kwds ): - """ - Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a - symbolic link (if found). Returns a triple of: env_script, base_path, real_version - """ - if version is None or self.versionless: - return self._find_dep_default( name, type=type, **kwds ) - else: - return self._find_dep_versioned( name, version, type=type, **kwds ) - - def _find_dep_versioned( self, name, version, type='package', **kwds ): - base_path = self.base_path - path = os.path.join( base_path, name, version ) - return self._galaxy_package_dep(path, version) - - def _find_dep_default( self, name, type='package', **kwds ): - base_path = self.base_path - path = os.path.join( base_path, name, 'default' ) - if os.path.islink( path ): - real_path = os.path.realpath( path ) - real_version = os.path.basename( real_path ) - return self._galaxy_package_dep(real_path, real_version) - else: - return INDETERMINATE_DEPENDENCY - - def _galaxy_package_dep( self, path, version ): - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version - return INDETERMINATE_DEPENDENCY - - -class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): - - def __init__(self, dependency_manager, **kwds): - super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) - - def _find_dep_versioned( self, name, version, type='package', **kwds ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) - base_path = self.base_path - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - return self._galaxy_package_dep(path, version) - else: - return INDETERMINATE_DEPENDENCY - - def _find_dep_default( self, name, type='package', **kwds ): - if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) - if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: - # Environment settings do not use versions. - return script, path, None - return INDETERMINATE_DEPENDENCY - - def _get_installed_dependency( self, name, type, version=None, **kwds ): - for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): - name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type - if version: - if name_and_type_equal and installed_tool_dependency.version == version: - return installed_tool_dependency - else: - if name_and_type_equal: - return installed_tool_dependency - return None - - def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): - tool_shed_repository = installed_tool_dependency.tool_shed_repository - return os.path.join( base_path, - name, - version, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) - - def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): - tool_shed_repository = installed_tool_dependency.tool_shed_repository - base_path = self.base_path - path = os.path.abspath( os.path.join( base_path, - 'environment_settings', - name, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) ) - if os.path.exists( path ): - script = os.path.join( path, 'env.sh' ) - return script, path, None - return INDETERMINATE_DEPENDENCY - - RESOLVER_CLASSES = { 'tool_shed_packages': ToolShedPackageDependencyResolver, 'galaxy_packages': GalaxyPackageDependencyResolver, diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/__init__.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -0,0 +1,18 @@ +from abc import ABCMeta, abstractmethod + +INDETERMINATE_DEPENDENCY = (None, None, None) + + +class DependencyResolver(object): + __metaclass__ = ABCMeta + + @abstractmethod + def resolve( self, name, version, type, **kwds ): + """ + Given inputs describing dependency in the abstract, yield tuple of + (script, bin, version). Here script is the env.sh file to source + before running a job, if that is not found the bin directory will be + appended to the path (if it is not None). Finally, version is the + resolved tool dependency version (which may differ from requested + version for instance if the request version is 'default'.) + """ diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -0,0 +1,51 @@ +from os.path import join, islink, realpath, basename, exists, abspath + +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY +from galaxy.util import string_as_bool + + +class GalaxyPackageDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager, **kwds): + ## Galaxy tool shed requires explicit versions on XML elements, + ## this in inconvient for testing or Galaxy instances not utilizing + ## the tool shed so allow a fallback version of the Galaxy package + ## resolver that will just grab 'default' version of exact version + ## unavailable. + self.versionless = string_as_bool(kwds.get('versionless', "false")) + self.base_path = abspath( kwds.get('base_path', dependency_manager.default_base_path) ) + + def resolve( self, name, version, type, **kwds ): + """ + Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a + symbolic link (if found). Returns a triple of: env_script, base_path, real_version + """ + if version is None or self.versionless: + return self._find_dep_default( name, type=type, **kwds ) + else: + return self._find_dep_versioned( name, version, type=type, **kwds ) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + base_path = self.base_path + path = join( base_path, name, version ) + return self._galaxy_package_dep(path, version) + + def _find_dep_default( self, name, type='package', **kwds ): + base_path = self.base_path + path = join( base_path, name, 'default' ) + if islink( path ): + real_path = realpath( path ) + real_version = basename( real_path ) + return self._galaxy_package_dep(real_path, real_version) + else: + return INDETERMINATE_DEPENDENCY + + def _galaxy_package_dep( self, path, version ): + script = join( path, 'env.sh' ) + if exists( script ): + return script, path, version + elif exists( join( path, 'bin' ) ): + return None, path, version + return INDETERMINATE_DEPENDENCY + +__all__ = [GalaxyPackageDependencyResolver] diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -0,0 +1,66 @@ +from os.path import abspath, join, exists + +from .galaxy_packages import GalaxyPackageDependencyResolver +from ..resolvers import INDETERMINATE_DEPENDENCY + + +class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + + def __init__(self, dependency_manager, **kwds): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) + base_path = self.base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) + return self._galaxy_package_dep(path, version) + else: + return INDETERMINATE_DEPENDENCY + + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) + if installed_tool_dependency: + script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if script and path: + # Environment settings do not use versions. + return script, path, None + return INDETERMINATE_DEPENDENCY + + def _get_installed_dependency( self, name, type, version=None, **kwds ): + installed_tool_dependencies = kwds.get("installed_tool_dependencies", []) + for installed_tool_dependency in (installed_tool_dependencies or []): + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type + if version: + if name_and_type_equal and installed_tool_dependency.version == version: + return installed_tool_dependency + else: + if name_and_type_equal: + return installed_tool_dependency + return None + + def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + return join( base_path, + name, + version, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) + + def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + base_path = self.base_path + path = abspath( join( base_path, + 'environment_settings', + name, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) ) + if exists( path ): + script = join( path, 'env.sh' ) + return script, path, None + return INDETERMINATE_DEPENDENCY + +__all__ = [ToolShedPackageDependencyResolver] https://bitbucket.org/galaxy/galaxy-central/commits/21b9e99a73fe/ Changeset: 21b9e99a73fe User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Introduce higher-level abstraction describing dependency. Passing the tuple (script, path, version) around is insufficient to represent other kinds of potential dependency load commands (namely module loads), this generalization allows more expressivity. I believe this abstraction also moves some logic out of the tool class that is better encapsulated in the deps module and submodules. Affected #: 5 files diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -40,7 +40,7 @@ from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction -from galaxy.tools.deps import DependencyManager +from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings from galaxy.tools.parameters.basic import (BaseURLToolParameter, DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter, @@ -2681,20 +2681,17 @@ installed_tool_dependencies = None for requirement in self.requirements: log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) - script_file = None - base_path = None - version = None + dependency = INDETERMINATE_DEPENDENCY if requirement.type in [ 'package', 'set_environment' ]: - script_file, base_path, version = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, - version=requirement.version, - type=requirement.type, - installed_tool_dependencies=installed_tool_dependencies ) - if script_file is None and base_path is None: + dependency = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, + version=requirement.version, + type=requirement.type, + installed_tool_dependencies=installed_tool_dependencies ) + dependency_commands = dependency.shell_commands( requirement ) + if not dependency_commands: log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) - elif requirement.type == 'package' and script_file is None: - commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; PATH="%s/bin:$PATH"; export PATH' % ( base_path, base_path ) ) else: - commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, script_file ) ) + commands.append(dependency_commands) return commands def build_redirect_url_params( self, param_dict ): """ diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/__init__.py --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -1,9 +1,7 @@ from abc import ABCMeta, abstractmethod -INDETERMINATE_DEPENDENCY = (None, None, None) - -class DependencyResolver(object): +class DependencyResolver( object ): __metaclass__ = ABCMeta @abstractmethod @@ -16,3 +14,21 @@ resolved tool dependency version (which may differ from requested version for instance if the request version is 'default'.) """ + + +class Dependency( object ): + __metaclass__ = ABCMeta + + @abstractmethod + def shell_commands( self, requirement ): + """ + Return shell commands to enable this dependency. + """ + + +class NullDependency( Dependency ): + + def shell_commands( self, requirement ): + return None + +INDETERMINATE_DEPENDENCY = NullDependency() diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -1,8 +1,11 @@ from os.path import join, islink, realpath, basename, exists, abspath -from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency from galaxy.util import string_as_bool +import logging +log = logging.getLogger( __name__ ) + class GalaxyPackageDependencyResolver(DependencyResolver): @@ -43,9 +46,28 @@ def _galaxy_package_dep( self, path, version ): script = join( path, 'env.sh' ) if exists( script ): - return script, path, version + return GalaxyPackageDependency(script, path, version) elif exists( join( path, 'bin' ) ): - return None, path, version + return GalaxyPackageDependency(None, path, version) return INDETERMINATE_DEPENDENCY -__all__ = [GalaxyPackageDependencyResolver] + +class GalaxyPackageDependency(Dependency): + + def __init__( self, script, path, version ): + self.script = script + self.path = path + self.version = version + + def shell_commands( self, requirement ): + base_path = self.path + if self.script is None and base_path is None: + log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) + commands = None + elif requirement.type == 'package' and self.script is None: + commands = 'PACKAGE_BASE=%s; export PACKAGE_BASE; PATH="%s/bin:$PATH"; export PATH' % ( base_path, base_path ) + else: + commands = 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, self.script ) + return commands + +__all__ = [GalaxyPackageDependencyResolver, GalaxyPackageDependency] diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -1,6 +1,6 @@ from os.path import abspath, join, exists -from .galaxy_packages import GalaxyPackageDependencyResolver +from .galaxy_packages import GalaxyPackageDependencyResolver, GalaxyPackageDependency from ..resolvers import INDETERMINATE_DEPENDENCY @@ -22,10 +22,10 @@ if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: + dependency = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if dependency.script and dependency.path: # Environment settings do not use versions. - return script, path, None + return GalaxyPackageDependency(dependency.script, dependency.path, None) return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, name, type, version=None, **kwds ): @@ -60,7 +60,7 @@ tool_shed_repository.installed_changeset_revision ) ) if exists( path ): script = join( path, 'env.sh' ) - return script, path, None + return GalaxyPackageDependency(script, path, None) return INDETERMINATE_DEPENDENCY __all__ = [ToolShedPackageDependencyResolver] diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -2,9 +2,11 @@ import os.path from os import makedirs, symlink from shutil import rmtree -from galaxy.tools.deps import DependencyManager +from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY +from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency from galaxy.util.bunch import Bunch from contextlib import contextmanager +from subprocess import Popen, PIPE def test_tool_dependencies(): @@ -24,24 +26,23 @@ __touch( os.path.join( p, "env.sh" ) ) dm = DependencyManager( default_base_path=base_path ) - d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) - assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) - assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) - assert d1_version == "1.0" - d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) - assert d2_script == None - assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) - assert d2_version == "2.0" + dependency = dm.find_dep( "dep1", "1.0" ) + assert dependency.script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert dependency.path == os.path.join( base_path, 'dep1', '1.0' ) + assert dependency.version == "1.0" + dependency = dm.find_dep( "dep1", "2.0" ) + assert dependency.script == None + assert dependency.path == os.path.join( base_path, 'dep1', '2.0' ) + assert dependency.version == "2.0" ## Test default versions symlink( os.path.join( base_path, 'dep1', '2.0'), os.path.join( base_path, 'dep1', 'default' ) ) - default_script, default_path, default_version = dm.find_dep( "dep1", None ) - assert default_version == "2.0" + dependency = dm.find_dep( "dep1", None ) + assert dependency.version == "2.0" ## Test default will not be fallen back upon by default - default_script, default_path, default_version = dm.find_dep( "dep1", "2.1" ) - assert default_script == None - assert default_version == None + dependency = dm.find_dep( "dep1", "2.1" ) + assert dependency == INDETERMINATE_DEPENDENCY TEST_REPO_USER = "devteam" @@ -56,9 +57,9 @@ dm = DependencyManager( default_base_path=base_path ) env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) os.makedirs(env_settings_dir) - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) - assert d1_version == None - assert d1_script == os.path.join(env_settings_dir, "env.sh"), d1_script + dependency = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) + assert dependency.version == None + assert dependency.script == os.path.join(env_settings_dir, "env.sh") def test_toolshed_package_requirements(): @@ -66,9 +67,9 @@ test_repo = __build_test_repo('package', version=TEST_VERSION) dm = DependencyManager( default_base_path=base_path ) package_dir = __build_ts_test_package(base_path) - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert d1_version == TEST_VERSION, d1_version - assert d1_script == os.path.join(package_dir, "env.sh"), d1_script + dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.version == TEST_VERSION + assert dependency.script == os.path.join(package_dir, "env.sh") def test_toolshed_tools_fallback_on_manual_dependencies(): @@ -76,9 +77,9 @@ dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") - d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) - assert d1_version == "1.0" - assert d1_script == env_path + dependency = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.version == "1.0" + assert dependency.script == env_path def test_toolshed_greater_precendence(): @@ -88,9 +89,9 @@ ts_package_dir = __build_ts_test_package(base_path) gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) ts_env_path = os.path.join(ts_package_dir, "env.sh") - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert d1_script != gx_env_path # Not the galaxy path, it should be the tool shed path used. - assert d1_script == ts_env_path + dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert dependency.script == ts_env_path def __build_ts_test_package(base_path, script_contents=''): @@ -99,6 +100,18 @@ return package_dir +def test_galaxy_dependency_object_script(): + with __test_base_path() as base_path: + ## Create env.sh file that just exports variable FOO and verify it + ## shell_commands export it correctly. + env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION, "export FOO=\"bar\"") + dependency = GalaxyPackageDependency(env_path, os.path.dirname(env_path), TEST_VERSION) + command = ["bash", "-c", "%s; echo \"$FOO\"" % dependency.shell_commands(Bunch(type="package"))] + process = Popen(command, stdout=PIPE) + output = process.communicate()[0].strip() + assert output == 'bar' + + def __setup_galaxy_package_dep(base_path, name, version, contents=""): dep_directory = os.path.join( base_path, name, version ) env_path = os.path.join( dep_directory, "env.sh" ) https://bitbucket.org/galaxy/galaxy-central/commits/31f6c565ca43/ Changeset: 31f6c565ca43 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Outline of integrating environment modules. This is completely untested and undoubtedly needs some fixes. The point is largely to outline how it could be done and let someone with interest in using this go in and flush out the implementation. Note: This is a community contributed feature and support from the core Galaxy team for it will be minimal. Affected #: 3 files diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -11,6 +11,7 @@ from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver +from .resolvers.modules import ModuleDependencyResolver class DependencyManager( object ): @@ -74,4 +75,5 @@ RESOLVER_CLASSES = { 'tool_shed_packages': ToolShedPackageDependencyResolver, 'galaxy_packages': GalaxyPackageDependencyResolver, + 'modules': ModuleDependencyResolver, } diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea lib/galaxy/tools/deps/resolvers/modules.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -0,0 +1,137 @@ +""" +This file contains the outline of an implementation to load environment modules +(http://modules.sourceforge.net/). + +This is a community contributed feature and the core Galaxy team does utilize +it, hence support for it will be minimal. The Galaxy team eagerly welcomes +community contribution and maintenance however. +""" +from os.path import exists, isdir, join +from StringIO import StringIO +from subprocess import Popen, PIPE + +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency +from galaxy.util import string_as_bool + +import logging +log = logging.getLogger( __name__ ) + + +DEFAULT_MODULE_COMMAND = 'module' +DEFAULT_MODULE_DIRECTORY = '/usr/share/modules/modulefiles' +DEFAULT_INDICATOR = '(default)' +DEFAULT_MODULE_PREFETCH = "true" +UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" + + +class ModuleDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager, **kwds): + self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) + self.versionless = string_as_bool(kwds.get('versionless', 'false')) + find_by = kwds.get('find_by', 'avail') + prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) + if find_by == 'directory': + directory = kwds.get('directory', DEFAULT_MODULE_DIRECTORY) + self.module_checker = DirectoryModuleChecker(self, directory, prefetch) + elif find_by == 'avail': + self.module_checker = AvailModuleChecker(self, prefetch) + else: + raise Exception(UNKNOWN_FIND_BY_MESSAGE % (find_by, ["avail", "directory"])) + + def resolve( self, name, version, type, **kwds ): + if type != "package": + return INDETERMINATE_DEPENDENCY + + if self.versionless: + version = None + + if self.__has_module(name, version): + return ModuleDependency(self, name, version) + + return INDETERMINATE_DEPENDENCY + + def __has_module(self, name, version): + return self.module_checker.has_module(name, version) + + +class DirectoryModuleChecker(object): + + def __init__(self, module_dependency_resolver, directory, prefetch): + self.module_dependency_resolver = module_dependency_resolver + self.directory = directory + if prefetch: + log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") + pass + + def has_module(self, module, version): + module_directory = join(self.directory, module) + has_module_directory = isdir( join( self.directory, module ) ) + if not version: + has_module = has_module_directory + else: + modulefile = join( module_directory, version ) + has_modulefile = exists( modulefile ) + has_module = has_module_directory and has_modulefile + return has_module + + +class AvailModuleChecker(object): + + def __init__(self, module_dependency_resolver, prefetch): + self.module_dependency_resolver = module_dependency_resolver + if prefetch: + prefetched_modules = [] + for module in self.__modules(): + prefetched_modules.append(module) + else: + prefetched_modules = None + self.prefetched_modules = prefetched_modules + + def has_module(self, module, version): + module_generator = self.prefetched_modules + if module_generator is None: + module_generator = self.__modules() + + for module_name, module_version in module_generator: + names_match = module == module_name + module_match = names_match and (version == None or module_version == version) + if module_match: + return True + return False + + def __modules(self): + raw_output = self.__module_avail_ouptut() + for line in StringIO(raw_output): + line = line and line.strip() + if not line or line.startswith("-"): + continue + + line_modules = line.split() + for module in line_modules: + if module.endswith(DEFAULT_INDICATOR): + module = module[0:-len(DEFAULT_INDICATOR)].strip() + module_parts = module.split('/') + module_version = None + if len(module_parts) == 2: + module_version = module_parts[1] + module_name = module_parts[0] + yield module_name, module_version + + def __module_avail_ouptut(self): + avail_command = '%s avail' % self.module_dependency_resolver.module_command + return Popen([avail_command], shell=True, stderr=PIPE).communicate()[1] + + +class ModuleDependency(Dependency): + + def __init__(self, module_dependency_resolver, module_name, module_version=None): + self.module_dependency_resolver = module_dependency_resolver + self.module_name = module_name + self.module_version = module_version + + def shell_commands(self, requirement): + command = '%s load %s' % (self.module_dependency_resolver.module_command, self.module_name) + if self.module_version: + command = '%s/%s' % self.module_version + return command diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,9 +1,11 @@ import tempfile import os.path -from os import makedirs, symlink +from stat import S_IXUSR +from os import makedirs, symlink, stat, chmod from shutil import rmtree from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency +from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver from galaxy.util.bunch import Bunch from contextlib import contextmanager from subprocess import Popen, PIPE @@ -100,6 +102,50 @@ return package_dir +def test_module_dependency_resolver(): + with __test_base_path() as temp_directory: + module_script = os.path.join(temp_directory, "module") + with open(module_script, 'w') as f: + f.write('''#!/bin/sh +cat %s/example_output 1>&2; +''' % temp_directory) + with open(os.path.join(temp_directory, "example_output"), "w") as f: + # Subset of module avail from MSI cluster. + f.write(''' +-------------------------- /soft/modules/modulefiles --------------------------- +JAGS/3.2.0-gcc45 +JAGS/3.3.0-gcc4.7.2 +ProbABEL/0.1-3 +ProbABEL/0.1-9e +R/2.12.2 +R/2.13.1 +R/2.14.1 +R/2.15.0 +R/2.15.1 +R/3.0.1(default) +abokia-blast/2.0.2-130524/ompi_intel +abokia-blast/2.0.2-130630/ompi_intel + +--------------------------- /soft/intel/modulefiles ---------------------------- +advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017 +advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 +advisor/2013/update3 intel/12.0 mkl/10.2.7.041 +''') + st = os.stat(module_script) + chmod(module_script, st.st_mode | S_IXUSR) + resolver = ModuleDependencyResolver(None, command=module_script) + module = resolver.resolve( name="R", version=None, type="package" ) + assert module.module_name == "R" + assert module.module_version == None + + module = resolver.resolve( name="R", version="3.0.1", type="package" ) + assert module.module_name == "R" + assert module.module_version == "3.0.1" + + module = resolver.resolve( name="R", version="3.0.4", type="package" ) + assert module == INDETERMINATE_DEPENDENCY + + def test_galaxy_dependency_object_script(): with __test_base_path() as base_path: ## Create env.sh file that just exports variable FOO and verify it @@ -189,11 +235,30 @@ # Unspecified base_paths are both default_base_paths assert dependency_resolvers[0].base_path == dependency_resolvers[1].base_path # Can specify custom base path... - assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy/" + assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy" # ... that is different from the default. assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path +def test_config_module_defaults(): + with __parse_resolvers('''<dependency_resolvers> + <modules /> +</dependency_resolvers> +''') as dependency_resolvers: + module_resolver = dependency_resolvers[0] + assert module_resolver.module_command == "module" + assert module_resolver.module_checker.__class__.__name__ == "AvailModuleChecker" + + +def test_config_module_directory_searcher(): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" directory="/opt/Modules/modulefiles" /> +</dependency_resolvers> +''') as dependency_resolvers: + module_resolver = dependency_resolvers[0] + assert module_resolver.module_checker.directory == "/opt/Modules/modulefiles" + + @contextmanager def __parse_resolvers(xml_content): with __test_base_path() as base_path: https://bitbucket.org/galaxy/galaxy-central/commits/54b9d1a2b1a9/ Changeset: 54b9d1a2b1a9 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Dynamically load tool dependency resolver plugins. This largely mirrors the code for the dynamic job runner so created common util code meant to be shared between dynamic job destinations and dynamic dependency resolves out into galaxy.util.submodules. TODO: Update job mapper to use this. Affected #: 5 files diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -8,10 +8,11 @@ log = logging.getLogger( __name__ ) from galaxy.util import parse_xml + from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver -from .resolvers.modules import ModuleDependencyResolver +from galaxy.util.submodules import submodules class DependencyManager( object ): @@ -35,6 +36,7 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) + self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) @@ -68,12 +70,19 @@ for resolver_element in resolvers_element.getchildren(): resolver_type = resolver_element.tag resolver_kwds = dict(resolver_element.items()) - resolver = RESOLVER_CLASSES[resolver_type](self, **resolver_kwds) + resolver = self.resolver_classes[resolver_type](self, **resolver_kwds) resolvers.append(resolver) return resolvers -RESOLVER_CLASSES = { - 'tool_shed_packages': ToolShedPackageDependencyResolver, - 'galaxy_packages': GalaxyPackageDependencyResolver, - 'modules': ModuleDependencyResolver, -} + def __resolvers_dict( self ): + resolver_dict = {} + for resolver_module in self.__resolver_modules(): + for clazz in resolver_module.__all__: + resolver_type = getattr(clazz, 'resolver_type', None) + if resolver_type: + resolver_dict[resolver_type] = clazz + return resolver_dict + + def __resolver_modules( self ): + import galaxy.tools.deps.resolvers + return submodules( galaxy.tools.deps.resolvers ) diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -8,6 +8,7 @@ class GalaxyPackageDependencyResolver(DependencyResolver): + resolver_type = "galaxy_packages" def __init__(self, dependency_manager, **kwds): ## Galaxy tool shed requires explicit versions on XML elements, diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -25,6 +25,7 @@ class ModuleDependencyResolver(DependencyResolver): + resolver_type = "modules" def __init__(self, dependency_manager, **kwds): self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) @@ -135,3 +136,5 @@ if self.module_version: command = '%s/%s' % self.module_version return command + +__all__ = [ModuleDependencyResolver] diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -5,6 +5,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + resolver_type = "tool_shed_packages" def __init__(self, dependency_manager, **kwds): super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/util/submodules.py --- /dev/null +++ b/lib/galaxy/util/submodules.py @@ -0,0 +1,30 @@ +from os import listdir +import logging +log = logging.getLogger( __name__ ) + + +def submodules( module ): + unsorted_submodule_names = __submodule_names( module ) + submodule_names = sorted( unsorted_submodule_names, reverse=True ) + submodules = [] + for submodule_name in submodule_names: + full_submodule = "%s.%s" % ( module.__name__, submodule_name ) + try: + __import__( full_submodule ) + submodule = getattr( module, submodule_name ) + submodules.append( submodule ) + except BaseException, exception: + exception_str = str( exception ) + message = "%s dynamic module could not be loaded: %s" % ( full_submodule, exception_str ) + log.debug( message ) + return submodules + + +def __submodule_names( module ): + module_dir = module.__path__[ 0 ] + names = [] + for fname in listdir( module_dir ): + if not( fname.startswith( "_" ) ) and fname.endswith( ".py" ): + submodule_name = fname[ :-len( ".py" ) ] + names.append( submodule_name ) + return names https://bitbucket.org/galaxy/galaxy-central/commits/6f54a6d1fc23/ Changeset: 6f54a6d1fc23 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Move requirement/dependency logic out of galaxy.tools and into galaxy.tools.deps, test. Moved ToolRequirement out into its own module along with parsing. Also moved all requirements handling logic into DependencyManager. Now the Tool class knows nothing about the internals or requirements or how to process dependencies. There is also greater unit test coverage for these concepts. Affected #: 4 files diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -41,6 +41,7 @@ from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY +from galaxy.tools.deps.requirements import parse_requirements_from_xml from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings from galaxy.tools.parameters.basic import (BaseURLToolParameter, DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter, @@ -938,15 +939,6 @@ def __iter__( self ): return iter( ( self.format, self.metadata_source, self.parent ) ) -class ToolRequirement( object ): - """ - Represents an external requirement that must be available for the tool to run (for example, a program, package, or library). - Requirements can optionally assert a specific version. - """ - def __init__( self, name=None, type=None, version=None ): - self.name = name - self.type = type - self.version = version class Tool( object, Dictifiable ): """ @@ -1236,10 +1228,7 @@ else: self.tests = None # Requirements (dependencies) - self.requirements = [] - requirements_elem = root.find( "requirements" ) - if requirements_elem: - self.parse_requirements( requirements_elem ) + self.requirements = parse_requirements_from_xml( root ) # Determine if this tool can be used in workflows self.is_workflow_compatible = self.check_workflow_compatible(root) # Trackster configuration. @@ -1810,17 +1799,6 @@ for name in param.get_dependencies(): context[ name ].refresh_on_change = True return param - def parse_requirements( self, requirements_elem ): - """ - Parse each requirement from the <requirements> element and add to - self.requirements - """ - for requirement_elem in requirements_elem.findall( 'requirement' ): - name = xml_text( requirement_elem ) - type = requirement_elem.get( "type", "package" ) - version = requirement_elem.get( "version", None ) - requirement = ToolRequirement( name=name, type=type, version=version ) - self.requirements.append( requirement ) def populate_tool_shed_info( self ): if self.repository_id is not None and 'ToolShedRepository' in self.app.model: @@ -2672,27 +2650,16 @@ command_line = command_line.replace(executable, abs_executable, 1) command_line = self.interpreter + " " + command_line return command_line + def build_dependency_shell_commands( self ): """Return a list of commands to be run to populate the current environment to include this tools requirements.""" - commands = [] if self.tool_shed_repository: installed_tool_dependencies = self.tool_shed_repository.installed_tool_dependencies else: installed_tool_dependencies = None - for requirement in self.requirements: - log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) - dependency = INDETERMINATE_DEPENDENCY - if requirement.type in [ 'package', 'set_environment' ]: - dependency = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, - version=requirement.version, - type=requirement.type, - installed_tool_dependencies=installed_tool_dependencies ) - dependency_commands = dependency.shell_commands( requirement ) - if not dependency_commands: - log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) - else: - commands.append(dependency_commands) - return commands + return self.app.toolbox.dependency_manager.dependency_shell_commands( self.requirements, + installed_tool_dependencies=installed_tool_dependencies ) + def build_redirect_url_params( self, param_dict ): """ Substitute parameter values into self.redirect_url_params diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -39,6 +39,22 @@ self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) + def dependency_shell_commands( self, requirements, **kwds ): + commands = [] + for requirement in requirements: + log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) + dependency = INDETERMINATE_DEPENDENCY + if requirement.type in [ 'package', 'set_environment' ]: + dependency = self.find_dep( name=requirement.name, + version=requirement.version, + type=requirement.type, + **kwds ) + dependency_commands = dependency.shell_commands( requirement ) + if not dependency_commands: + log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) + else: + commands.append( dependency_commands ) + return commands def find_dep( self, name, version=None, type='package', **kwds ): for resolver in self.dependency_resolvers: diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/deps/requirements.py --- /dev/null +++ b/lib/galaxy/tools/deps/requirements.py @@ -0,0 +1,57 @@ +from galaxy.util import xml_text + +DEFAULT_REQUIREMENT_TYPE = "package" +DEFAULT_REQUIREMENT_VERSION = None + + +class ToolRequirement( object ): + """ + Represents an external requirement that must be available for the tool to + run (for example, a program, package, or library). Requirements can + optionally assert a specific version. + """ + def __init__( self, name=None, type=None, version=None ): + self.name = name + self.type = type + self.version = version + + +def parse_requirements_from_xml( xml_root ): + """ + + >>> from galaxy.util import parse_xml + >>> from elementtree import ElementTree + >>> def load_requirements( contents ): + ... contents_document = '''<tool><requirements>%s</requirements></tool>''' + ... root = ElementTree.fromstring( contents_document % contents ) + ... return parse_requirements_from_xml( root ) + >>> reqs = load_requirements('''<requirement>bwa</requirement>''') + >>> reqs[0].name + 'bwa' + >>> reqs[0].version is None + True + >>> reqs[0].type + 'package' + >>> reqs = load_requirements('''<requirement type="binary" version="1.3.3">cufflinks</requirement>''') + >>> reqs[0].name + 'cufflinks' + >>> reqs[0].version + '1.3.3' + >>> reqs[0].type + 'binary' + """ + requirements_elem = xml_root.find( "requirements" ) + + requirement_elems = [] + if requirements_elem: + requirement_elems = requirements_elem.findall( 'requirement' ) + + requirements = [] + for requirement_elem in requirement_elems: + name = xml_text( requirement_elem ) + type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE ) + version = requirement_elem.get( "version", DEFAULT_REQUIREMENT_VERSION ) + requirement = ToolRequirement( name=name, type=type, version=version ) + requirements.append( requirement ) + + return requirements diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -152,10 +152,25 @@ ## shell_commands export it correctly. env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION, "export FOO=\"bar\"") dependency = GalaxyPackageDependency(env_path, os.path.dirname(env_path), TEST_VERSION) - command = ["bash", "-c", "%s; echo \"$FOO\"" % dependency.shell_commands(Bunch(type="package"))] - process = Popen(command, stdout=PIPE) - output = process.communicate()[0].strip() - assert output == 'bar' + __assert_foo_exported( dependency.shell_commands( Bunch( type="package" ) ) ) + + +def test_shell_commands_built(): + ## Test that dependency manager builds valid shell commands for a list of + ## requirements. + with __test_base_path() as base_path: + dm = DependencyManager( default_base_path=base_path ) + __setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" ) + mock_requirements = [ Bunch(type="package", version=TEST_VERSION, name=TEST_REPO_NAME ) ] + commands = dm.dependency_shell_commands( mock_requirements ) + __assert_foo_exported( commands ) + + +def __assert_foo_exported( commands ): + command = ["bash", "-c", "%s; echo \"$FOO\"" % "".join(commands)] + process = Popen(command, stdout=PIPE) + output = process.communicate()[0].strip() + assert output == 'bar' def __setup_galaxy_package_dep(base_path, name, version, contents=""): https://bitbucket.org/galaxy/galaxy-central/commits/dc9c0e065be0/ Changeset: dc9c0e065be0 User: sjguest Date: 2013-10-17 06:15:21 Summary: Fixed ModuleDependencyResolver, versionless fallback, and modulepath Affected #: 1 file diff -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae -r dc9c0e065be01f8d16b35eec7b53098fdfa0fd05 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -6,6 +6,7 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes community contribution and maintenance however. """ +from os import environ from os.path import exists, isdir, join from StringIO import StringIO from subprocess import Popen, PIPE @@ -17,8 +18,12 @@ log = logging.getLogger( __name__ ) -DEFAULT_MODULE_COMMAND = 'module' -DEFAULT_MODULE_DIRECTORY = '/usr/share/modules/modulefiles' +if environ.has_key('MODULEPATH'): + DEFAULT_MODULE_PATH = environ['MODULEPATH'] +elif environ.has_key('MODULESHOME'): + DEFAULT_MODULE_PATH = join(environ['MODULESHOME'], 'modulefiles') +else: + DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" @@ -28,13 +33,12 @@ resolver_type = "modules" def __init__(self, dependency_manager, **kwds): - self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) self.versionless = string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) if find_by == 'directory': - directory = kwds.get('directory', DEFAULT_MODULE_DIRECTORY) - self.module_checker = DirectoryModuleChecker(self, directory, prefetch) + modulepath = kwds.get('modulepath', DEFAULT_MODULE_PATH) + self.module_checker = DirectoryModuleChecker(self, modulepath, prefetch) elif find_by == 'avail': self.module_checker = AvailModuleChecker(self, prefetch) else: @@ -44,11 +48,10 @@ if type != "package": return INDETERMINATE_DEPENDENCY - if self.versionless: - version = None - if self.__has_module(name, version): return ModuleDependency(self, name, version) + elif self.versionless and self.__has_module(name, None): + return ModuleDependency(self, name, None) return INDETERMINATE_DEPENDENCY @@ -58,23 +61,26 @@ class DirectoryModuleChecker(object): - def __init__(self, module_dependency_resolver, directory, prefetch): + def __init__(self, module_dependency_resolver, modulepath, prefetch): self.module_dependency_resolver = module_dependency_resolver - self.directory = directory + self.directories = modulepath.split(':') if prefetch: log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") pass def has_module(self, module, version): - module_directory = join(self.directory, module) - has_module_directory = isdir( join( self.directory, module ) ) - if not version: - has_module = has_module_directory - else: - modulefile = join( module_directory, version ) - has_modulefile = exists( modulefile ) - has_module = has_module_directory and has_modulefile - return has_module + for directory in self.directories: + module_directory = join(directory, module) + has_module_directory = isdir( module_directory ) + if not version: + has_module = has_module_directory or exists(module_directory) # could be a bare modulefile + else: + modulefile = join( module_directory, version ) + has_modulefile = exists( modulefile ) + has_module = has_module_directory and has_modulefile + if has_module: + return True + return False class AvailModuleChecker(object): @@ -102,7 +108,7 @@ return False def __modules(self): - raw_output = self.__module_avail_ouptut() + raw_output = self.__module_avail_output() for line in StringIO(raw_output): line = line and line.strip() if not line or line.startswith("-"): @@ -119,10 +125,9 @@ module_name = module_parts[0] yield module_name, module_version - def __module_avail_ouptut(self): - avail_command = '%s avail' % self.module_dependency_resolver.module_command - return Popen([avail_command], shell=True, stderr=PIPE).communicate()[1] - + def __module_avail_output(self): + avail_command = ['modulecmd', 'sh', 'avail'] + return Popen(avail_command, stderr=PIPE).communicate()[1] class ModuleDependency(Dependency): @@ -132,9 +137,10 @@ self.module_version = module_version def shell_commands(self, requirement): - command = '%s load %s' % (self.module_dependency_resolver.module_command, self.module_name) + module_to_load = self.module_name if self.module_version: - command = '%s/%s' % self.module_version + module_to_load = '%s/%s' % (self.module_name, self.module_version) + command = 'eval `modulecmd sh load %s`' % (module_to_load) return command __all__ = [ModuleDependencyResolver] https://bitbucket.org/galaxy/galaxy-central/commits/93286e74e18b/ Changeset: 93286e74e18b User: jmchilton Date: 2013-10-17 06:15:21 Summary: Small touch ups to module code enhancements. Affected #: 1 file diff -r dc9c0e065be01f8d16b35eec7b53098fdfa0fd05 -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -6,7 +6,7 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes community contribution and maintenance however. """ -from os import environ +from os import environ, pathsep from os.path import exists, isdir, join from StringIO import StringIO from subprocess import Popen, PIPE @@ -18,12 +18,7 @@ log = logging.getLogger( __name__ ) -if environ.has_key('MODULEPATH'): - DEFAULT_MODULE_PATH = environ['MODULEPATH'] -elif environ.has_key('MODULESHOME'): - DEFAULT_MODULE_PATH = join(environ['MODULESHOME'], 'modulefiles') -else: - DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' +DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" @@ -44,6 +39,15 @@ else: raise Exception(UNKNOWN_FIND_BY_MESSAGE % (find_by, ["avail", "directory"])) + def __default_modulespath(self): + if 'MODULEPATH' in environ: + module_path = environ['MODULEPATH'] + elif 'MODULESHOME' in environ: + module_path = join(environ['MODULESHOME'], 'modulefiles') + else: + module_path = DEFAULT_MODULE_PATH + return module_path + def resolve( self, name, version, type, **kwds ): if type != "package": return INDETERMINATE_DEPENDENCY @@ -63,24 +67,25 @@ def __init__(self, module_dependency_resolver, modulepath, prefetch): self.module_dependency_resolver = module_dependency_resolver - self.directories = modulepath.split(':') + self.directories = modulepath.split(pathsep) if prefetch: log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") pass def has_module(self, module, version): + has_module = False for directory in self.directories: module_directory = join(directory, module) has_module_directory = isdir( module_directory ) if not version: - has_module = has_module_directory or exists(module_directory) # could be a bare modulefile + has_module = has_module_directory or exists(module_directory) # could be a bare modulefile else: modulefile = join( module_directory, version ) has_modulefile = exists( modulefile ) has_module = has_module_directory and has_modulefile if has_module: - return True - return False + break + return has_module class AvailModuleChecker(object): @@ -129,6 +134,7 @@ avail_command = ['modulecmd', 'sh', 'avail'] return Popen(avail_command, stderr=PIPE).communicate()[1] + class ModuleDependency(Dependency): def __init__(self, module_dependency_resolver, module_name, module_version=None): https://bitbucket.org/galaxy/galaxy-central/commits/66222d30c816/ Changeset: 66222d30c816 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Add unit tests for module behavior contributed by Simon Guest. Affected #: 2 files diff -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d -r 66222d30c81606b57d09e195365707c9f61b2946 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -17,7 +17,7 @@ import logging log = logging.getLogger( __name__ ) - +DEFAULT_MODULECMD_PATH = "modulecmd" # Just check path DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" @@ -31,8 +31,9 @@ self.versionless = string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) + self.modulecmd = kwds.get('modulecmd', DEFAULT_MODULECMD_PATH) if find_by == 'directory': - modulepath = kwds.get('modulepath', DEFAULT_MODULE_PATH) + modulepath = kwds.get('modulepath', self.__default_modulespath()) self.module_checker = DirectoryModuleChecker(self, modulepath, prefetch) elif find_by == 'avail': self.module_checker = AvailModuleChecker(self, prefetch) @@ -131,7 +132,7 @@ yield module_name, module_version def __module_avail_output(self): - avail_command = ['modulecmd', 'sh', 'avail'] + avail_command = [self.module_dependency_resolver.modulecmd, 'sh', 'avail'] return Popen(avail_command, stderr=PIPE).communicate()[1] @@ -146,7 +147,7 @@ module_to_load = self.module_name if self.module_version: module_to_load = '%s/%s' % (self.module_name, self.module_version) - command = 'eval `modulecmd sh load %s`' % (module_to_load) + command = 'eval `%s sh load %s`' % (self.module_dependency_resolver.modulecmd, module_to_load) return command __all__ = [ModuleDependencyResolver] diff -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d -r 66222d30c81606b57d09e195365707c9f61b2946 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,11 +1,11 @@ import tempfile import os.path from stat import S_IXUSR -from os import makedirs, symlink, stat, chmod +from os import makedirs, stat, symlink, chmod, environ from shutil import rmtree from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency -from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver +from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver, ModuleDependency from galaxy.util.bunch import Bunch from contextlib import contextmanager from subprocess import Popen, PIPE @@ -92,7 +92,7 @@ gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) ts_env_path = os.path.join(ts_package_dir, "env.sh") dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. assert dependency.script == ts_env_path @@ -104,9 +104,8 @@ def test_module_dependency_resolver(): with __test_base_path() as temp_directory: - module_script = os.path.join(temp_directory, "module") - with open(module_script, 'w') as f: - f.write('''#!/bin/sh + module_script = os.path.join(temp_directory, "modulecmd") + __write_script(module_script, '''#!/bin/sh cat %s/example_output 1>&2; ''' % temp_directory) with open(os.path.join(temp_directory, "example_output"), "w") as f: @@ -131,9 +130,7 @@ advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 advisor/2013/update3 intel/12.0 mkl/10.2.7.041 ''') - st = os.stat(module_script) - chmod(module_script, st.st_mode | S_IXUSR) - resolver = ModuleDependencyResolver(None, command=module_script) + resolver = ModuleDependencyResolver(None, modulecmd=module_script) module = resolver.resolve( name="R", version=None, type="package" ) assert module.module_name == "R" assert module.module_version == None @@ -146,6 +143,31 @@ assert module == INDETERMINATE_DEPENDENCY +def test_module_dependency(): + with __test_base_path() as temp_directory: + ## Create mock modulecmd script that just exports a variable + ## the way modulecmd sh load would, but also validate correct + ## module name and version are coming through. + mock_modulecmd = os.path.join(temp_directory, 'modulecmd') + __write_script(mock_modulecmd, '''#!/bin/sh +if [ $3 != "foomodule/1.0" ]; +then + exit 1 +fi +echo 'FOO="bar"' +''') + resolver = Bunch(modulecmd=mock_modulecmd) + dependency = ModuleDependency(resolver, "foomodule", "1.0") + __assert_foo_exported( dependency.shell_commands( Bunch( type="package" ) ) ) + + +def __write_script(path, contents): + with open(path, 'w') as f: + f.write(contents) + st = stat(path) + chmod(path, st.st_mode | S_IXUSR) + + def test_galaxy_dependency_object_script(): with __test_base_path() as base_path: ## Create env.sh file that just exports variable FOO and verify it @@ -170,7 +192,7 @@ command = ["bash", "-c", "%s; echo \"$FOO\"" % "".join(commands)] process = Popen(command, stdout=PIPE) output = process.communicate()[0].strip() - assert output == 'bar' + assert output == 'bar', "Command %s exports FOO as %s, not bar" % (command, output) def __setup_galaxy_package_dep(base_path, name, version, contents=""): @@ -261,17 +283,68 @@ </dependency_resolvers> ''') as dependency_resolvers: module_resolver = dependency_resolvers[0] - assert module_resolver.module_command == "module" assert module_resolver.module_checker.__class__.__name__ == "AvailModuleChecker" +def test_config_modulepath(): + # Test reads and splits MODULEPATH if modulepath is not specified. + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" modulepath="/opt/modules/modulefiles:/usr/local/modules/modulefiles" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles", "/usr/local/modules/modulefiles"] + + +def test_config_MODULEPATH(): + # Test reads and splits MODULEPATH if modulepath is not specified. + with __environ({"MODULEPATH": "/opt/modules/modulefiles:/usr/local/modules/modulefiles"}): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles", "/usr/local/modules/modulefiles"] + + +def test_config_MODULESHOME(): + # Test fallbacks to read MODULESHOME if modulepath is not specified and + # neither is MODULEPATH. + with __environ({"MODULESHOME": "/opt/modules"}, remove="MODULEPATH"): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles"] + + def test_config_module_directory_searcher(): with __parse_resolvers('''<dependency_resolvers> - <modules find_by="directory" directory="/opt/Modules/modulefiles" /> + <modules find_by="directory" modulepath="/opt/Modules/modulefiles" /></dependency_resolvers> ''') as dependency_resolvers: module_resolver = dependency_resolvers[0] - assert module_resolver.module_checker.directory == "/opt/Modules/modulefiles" + assert module_resolver.module_checker.directories == ["/opt/Modules/modulefiles"] + + +@contextmanager +def __environ(values, remove=[]): + """ + Modify the environment for a test, adding/updating values in dict `values` and + removing any environment variables mentioned in list `remove`. + """ + new_keys = set(environ.keys()) - set(values.keys()) + old_environ = environ.copy() + try: + environ.update(values) + for to_remove in remove: + try: + del environ[remove] + except KeyError: + pass + yield + finally: + environ.update(old_environ) + for key in new_keys: + del environ[key] @contextmanager @@ -282,4 +355,3 @@ f.flush() dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) yield dm.dependency_resolvers - https://bitbucket.org/galaxy/galaxy-central/commits/a96f6548bfc3/ Changeset: a96f6548bfc3 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Add method to DependencyManager determining if tool shed dependencies used. Affected #: 2 files diff -r 66222d30c81606b57d09e195365707c9f61b2946 -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -56,6 +56,9 @@ commands.append( dependency_commands ) return commands + def uses_tool_shed_dependencies(self): + return any( map( lambda r: isinstance( r, ToolShedPackageDependencyResolver ), self.dependency_resolvers ) ) + def find_dep( self, name, version=None, type='package', **kwds ): for resolver in self.dependency_resolvers: dependency = resolver.resolve( name, version, type, **kwds ) diff -r 66222d30c81606b57d09e195365707c9f61b2946 -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -277,6 +277,20 @@ assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path +def test_uses_tool_shed_dependencies(): + with __dependency_manager('''<dependency_resolvers> + <galaxy_packages /> +</dependency_resolvers> +''') as dm: + assert not dm.uses_tool_shed_dependencies() + + with __dependency_manager('''<dependency_resolvers> + <tool_shed_packages /> +</dependency_resolvers> +''') as dm: + assert dm.uses_tool_shed_dependencies() + + def test_config_module_defaults(): with __parse_resolvers('''<dependency_resolvers><modules /> @@ -349,9 +363,15 @@ @contextmanager def __parse_resolvers(xml_content): + with __dependency_manager(xml_content) as dm: + yield dm.dependency_resolvers + + +@contextmanager +def __dependency_manager(xml_content): with __test_base_path() as base_path: f = tempfile.NamedTemporaryFile() f.write(xml_content) f.flush() dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) - yield dm.dependency_resolvers + yield dm https://bitbucket.org/galaxy/galaxy-central/commits/8e001dc9675c/ Changeset: 8e001dc9675c User: jmchilton Date: 2013-10-17 06:15:21 Summary: Dependencies: Do not attempt to install package dependencies in tool shed if the will be ignored. That is if dependency resolvers are configured to ignore tool shed dependencies as outlined by Simon Guest, though stopping short of marking these dependencies as installed - they will still be marked as errors for now. TODO: Determine what to do with dependencies that can be resolved (just mark as installed? a brand new state? mark as never installed?). Affected #: 1 file diff -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f -r 8e001dc9675c105d83d83f6d5a756c11759e8bb2 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -6,6 +6,7 @@ from galaxy import util from galaxy import web from galaxy.util import json +from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import container_util @@ -457,7 +458,19 @@ tool_dependency = tool_dependencies[ index ] if tool_dependency.can_install: try: - tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) + dependencies_ignored = app.toolbox.dependency_manager and not app.toolbox.dependency_manager.uses_tool_shed_dependencies() + if dependencies_ignored: + log.info("Skipping package %s, tool shed dependency resolver not enabled." % package_name) + # Tool dependency resolves have been configured and they do not + # include the tool shed. Do not install package. + status = app.model.ToolDependency.installation_status.ERROR + if app.toolbox.dependency_manager.find_dep( package_name, package_version, type='package') != INDETERMINATE_DEPENDENCY: + ## TODO: Do something here such as marking it installed or + ## configured externally. + pass + tool_dependency.status = status + else: + tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) except Exception, e: error_message = "Error installing tool dependency %s version %s: %s" % ( str( package_name ), str( package_version ), str( e ) ) log.exception( error_message ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Tool shed install move_file bug fix (divergence between 67be847 and 8f2b113).
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ebd092733e55/ Changeset: ebd092733e55 User: jmchilton Date: 2013-10-17 06:00:55 Summary: Tool shed install move_file bug fix (divergence between 67be847 and 8f2b113). Affected #: 1 file diff -r 7c05567cdc1b9d1c08e59c2927fb9e85009262e4 -r ebd092733e552ef6a846d55b69d22dc779dfcc5f lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -560,8 +560,8 @@ # <source>misc/some_file</source> # <destination>$INSTALL_DIR/bin</destination> # </action> - action_dict[ 'source' ] = evaluate_template( action_elem.find( 'source' ).text ) - action_dict[ 'destination' ] = evaluate_template( action_elem.find( 'destination' ).text ) + action_dict[ 'source' ] = td_common_util.evaluate_template( action_elem.find( 'source' ).text, install_dir ) + action_dict[ 'destination' ] = td_common_util.evaluate_template( action_elem.find( 'destination' ).text, install_dir ) action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' ) elif action_type == 'set_environment': # <action type="set_environment"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/913f3ab84629/ Changeset: 913f3ab84629 Branch: abrenner/histogram2xml-python-module-rpy-requires-1379010965302 User: abrenner Date: 2013-09-12 20:36:20 Summary: histogram2.xml python module rpy requires R package. Affected #: 1 file diff -r b9505d30f85f6e5e49640c6428185ab427899ec4 -r 913f3ab84629b1c2dff39a8788e1e0201f82ecaa tools/plotting/histogram2.xml --- a/tools/plotting/histogram2.xml +++ b/tools/plotting/histogram2.xml @@ -27,6 +27,7 @@ </tests><requirements><requirement type="python-module">rpy</requirement> + <requirement type="package">R</requirement></requirements><help> https://bitbucket.org/galaxy/galaxy-central/commits/f3e736fe03df/ Changeset: f3e736fe03df User: abrenner Date: 2013-09-12 20:39:39 Summary: Merged in abrenner/histogram2xml-python-module-rpy-requires-1379010965302 (pull request #1) histogram2.xml python module rpy requires R package. Affected #: 1 file diff -r b9505d30f85f6e5e49640c6428185ab427899ec4 -r f3e736fe03df3a6dd5438c12ba35ea791a1eaca9 tools/plotting/histogram2.xml --- a/tools/plotting/histogram2.xml +++ b/tools/plotting/histogram2.xml @@ -27,6 +27,7 @@ </tests><requirements><requirement type="python-module">rpy</requirement> + <requirement type="package">R</requirement></requirements><help> https://bitbucket.org/galaxy/galaxy-central/commits/7c05567cdc1b/ Changeset: 7c05567cdc1b User: jmchilton Date: 2013-10-17 05:12:40 Summary: Merged in abrenner/galaxy-central (pull request #215) histogram2 - RPY requires R - missing from requirements tag Thanks for the contribution! Affected #: 1 file diff -r 3825c8d493649a72c6fc65bdd06b2362f0bf5930 -r 7c05567cdc1b9d1c08e59c2927fb9e85009262e4 tools/plotting/histogram2.xml --- a/tools/plotting/histogram2.xml +++ b/tools/plotting/histogram2.xml @@ -27,6 +27,7 @@ </tests><requirements><requirement type="python-module">rpy</requirement> + <requirement type="package">R</requirement></requirements><help> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/432999eabbaa/ Changeset: 432999eabbaa User: jmchilton Date: 2013-10-17 00:37:37 Summary: Add job_script module from LWR including GALAXY_SLOTS logic. Use new job_script module in the DRMAA runner. This implements the long discussed GALAXY_SLOTS logic for PBS/TORQUE, SLURM, and grid engine when coming through the DRMAA runner. Affected #: 6 files diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -15,6 +15,7 @@ from galaxy.jobs.command_factory import build_command from galaxy import model from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size +from galaxy.jobs.runners.util.job_script import job_script log = logging.getLogger( __name__ ) @@ -218,6 +219,16 @@ external_metadata_proc.wait() log.debug( 'execution of external set_meta for job %d finished' % job_wrapper.job_id ) + def get_job_file(self, job_wrapper, **kwds): + options = dict( + galaxy_lib=job_wrapper.galaxy_lib_dir, + env_setup_commands=job_wrapper.get_env_setup_clause(), + working_directory=os.path.abspath( job_wrapper.working_directory ), + command=job_wrapper.runner_command_line, + ) + options.update(**kwds) + return job_script(**options) + class AsynchronousJobState( object ): """ diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -37,29 +37,10 @@ drmaa.JobState.FAILED: 'job finished, but failed', } -# The last four lines (following the last fi) will: -# - setup the env -# - move to the job wrapper's working directory -# - execute the command -# - take the command's exit code ($?) and write it to a file. -drm_template = """#!/bin/sh -GALAXY_LIB="%s" -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -%s -cd %s -%s -echo $? > %s -""" DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', - 'jobName','email','project' ] + 'jobName', 'email', 'project' ] + class DRMAAJobRunner( AsynchronousJobRunner ): """ @@ -138,12 +119,7 @@ jt.nativeSpecification = native_spec # fill in the DRM's job run template - script = drm_template % ( job_wrapper.galaxy_lib_dir, - job_wrapper.get_env_setup_clause(), - os.path.abspath( job_wrapper.working_directory ), - command_line, - ajs.exit_code_file ) - + script = self.get_job_file(job_wrapper, exit_code_path=ajs.exit_code_file) try: fh = file( ajs.job_file, "w" ) fh.write( script ) diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/util/__init__.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/__init__.py @@ -0,0 +1,9 @@ +""" +This module and its submodules contains utilities for running external +processes and interfacing with job managers. This module should contain +functionality shared between Galaxy and the LWR. +""" +try: + from galaxy.util.bunch import Bunch +except ImportError: + from lwr.util import Bunch diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh --- /dev/null +++ b/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh @@ -0,0 +1,11 @@ +export GALAXY_SLOTS_CONFIGURED="1" +if [ -n "$SLURM_JOB_NUM_NODES" ]; then + GALAXY_SLOTS="$SLURM_JOB_NUM_NODES" +elif [ -n "$NSLOTS" ]; then + GALAXY_SLOTS="$NSLOTS" +elif [ -f "$PBS_NODEFILE" ]; then + GALAXY_SLOTS=`wc -l < $PBS_NODEFILE` +else + GALAXY_SLOTS="1" + unset GALAXY_SLOTS_CONFIGURED +fi diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh --- /dev/null +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -0,0 +1,17 @@ +#!/bin/sh +$headers +$slots_statement +export GALAXY_SLOTS +GALAXY_LIB="$galaxy_lib" +if [ "$GALAXY_LIB" != "None" ]; then + if [ -n "$PYTHONPATH" ]; then + PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" + else + PYTHONPATH="$GALAXY_LIB" + fi + export PYTHONPATH +fi +$env_setup_commands +cd $working_directory +$command +echo $? > $exit_code_path diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/util/job_script/__init__.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -0,0 +1,54 @@ +from string import Template +from pkg_resources import resource_string + +DEFAULT_JOB_FILE_TEMPLATE = Template( + resource_string(__name__, 'DEFAULT_JOB_FILE_TEMPLATE.sh') +) + +SLOTS_STATEMENT_CLUSTER_DEFAULT = \ + resource_string(__name__, 'CLUSTER_SLOTS_STATEMENT.sh') + +SLOTS_STATEMENT_SINGLE = """ +GALAXY_SLOTS="1" +""" + +REQUIRED_TEMPLATE_PARAMS = ['working_directory', 'command', 'exit_code_path'] +OPTIONAL_TEMPLATE_PARAMS = { + 'galaxy_lib': None, + 'headers': '', + 'env_setup_commands': '', + 'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT, +} + + +def job_script(template=DEFAULT_JOB_FILE_TEMPLATE, **kwds): + """ + + >>> has_exception = False + >>> try: job_script() + ... except Exception as e: has_exception = True + >>> has_exception + True + >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec') + >>> '\\nuptime\\n' in script + True + >>> 'echo $? > ec' in script + True + >>> 'GALAXY_LIB="None"' in script + True + >>> script.startswith('#!/bin/sh\\n#PBS -test\\n') + False + >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', headers='#PBS -test') + >>> script.startswith('#!/bin/sh\\n#PBS -test\\n') + True + >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', slots_statement='GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"') + >>> script.find('GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"\\nexport GALAXY_SLOTS\\n') > 0 + True + """ + if any([param not in kwds for param in REQUIRED_TEMPLATE_PARAMS]): + raise Exception("Failed to create job_script, a required parameter is missing.") + template_params = OPTIONAL_TEMPLATE_PARAMS.copy() + template_params.update(**kwds) + if not isinstance(template, Template): + template = Template(template) + return template.safe_substitute(template_params) https://bitbucket.org/galaxy/galaxy-central/commits/126d5c5b9a49/ Changeset: 126d5c5b9a49 User: jmchilton Date: 2013-10-17 00:37:37 Summary: Condor job runner enhancements. Implement exit code handling and submission script parameters (job destination parameter <param id="submit_xxxx">yyyy</param> will be passed through as xxxx = yyyy. Bring in LWR helper scripts to implement most of these and clean up things. Bring in LWR external.py for parsing ID's comding from resource managers, seems like overkill for now in Galaxy because just used in condor, but in LWR also used for its version of the CLI runners (will merge those changes in down the road). Utilize job_scripts module for building script (this is what fixes exit code handling). TODO: GALAXY_SLOTS logic still not implemented, but should be easier now. Affected #: 3 files diff -r 432999eabbaa97528a2bd37f1b99de86b39a924f -r 126d5c5b9a4938909b518cb176dc47890e1c13e5 lib/galaxy/jobs/runners/condor.py --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -3,40 +3,18 @@ """ import os -import re -import sys -import time import logging -import subprocess from galaxy import model from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner - -from galaxy.util import asbool +from galaxy.jobs.runners.util.condor import submission_params, build_submit_description +from galaxy.jobs.runners.util.condor import condor_submit, condor_stop +from galaxy.jobs.runners.util.condor import summarize_condor_log log = logging.getLogger( __name__ ) __all__ = [ 'CondorJobRunner' ] -drm_template = """#!/bin/sh -GALAXY_LIB="%s" -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -cd %s -%s -""" - -default_query_classad = dict( - universe = 'vanilla', - getenv = 'true', - notification = 'NEVER', -) class CondorJobState( AsynchronousJobState ): def __init__( self, **kwargs ): @@ -49,21 +27,22 @@ self.user_log = None self.user_log_size = 0 + class CondorJobRunner( AsynchronousJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ runner_name = "CondorRunner" + def __init__( self, app, nworkers ): """Initialize this job runner and start the monitor thread""" super( CondorJobRunner, self ).__init__( app, nworkers ) self._init_monitor_thread() self._init_worker_threads() - # superclass url_to_destination is fine - condor runner does not take params - def queue_job( self, job_wrapper ): """Create job script and submit it to the DRM""" + # prepare the job if not self.prepare_job( job_wrapper, include_metadata=True ): return @@ -78,25 +57,30 @@ galaxy_id_tag = job_wrapper.get_id_tag() # get destination params - query_params = default_query_classad.copy() - query_params.update( job_destination.params ) + query_params = submission_params(**job_destination.params) # define job attributes - cjs = CondorJobState( files_dir=self.app.config.cluster_files_directory, job_wrapper=job_wrapper ) - cjs.user_log = os.path.join( self.app.config.cluster_files_directory, 'galaxy_%s.condor.log' % galaxy_id_tag ) + cjs = CondorJobState( + files_dir=self.app.config.cluster_files_directory, + job_wrapper=job_wrapper + ) + + cluster_directory = self.app.config.cluster_files_directory + cjs.user_log = os.path.join( cluster_directory, 'galaxy_%s.condor.log' % galaxy_id_tag ) cjs.register_cleanup_file_attribute( 'user_log' ) - submit_file = os.path.join( self.app.config.cluster_files_directory, 'galaxy_%s.condor.desc' % galaxy_id_tag ) + submit_file = os.path.join( cluster_directory, 'galaxy_%s.condor.desc' % galaxy_id_tag ) executable = cjs.job_file - submit_desc = [ ] - for k, v in query_params.items(): - submit_desc.append( '%s = %s' % ( k, v ) ) - submit_desc.append( 'executable = ' + executable ) - submit_desc.append( 'output = ' + cjs.output_file ) - submit_desc.append( 'error = ' + cjs.error_file ) - submit_desc.append( 'log = ' + cjs.user_log ) - submit_desc.append( 'queue' ) - script = drm_template % (job_wrapper.galaxy_lib_dir, os.path.abspath( job_wrapper.working_directory ), command_line) + build_submit_params = dict( + executable=executable, + output=cjs.output_file, + error=cjs.error_file, + user_log=cjs.user_log, + query_params=query_params, + ) + + submit_file_contents = build_submit_description(**build_submit_params) + script = self.get_job_file( job_wrapper, exit_code_path=cjs.exit_code_file ) try: fh = file( executable, "w" ) fh.write( script ) @@ -108,10 +92,7 @@ return try: - fh = file( submit_file, 'w' ) - for line in submit_desc: - fh.write( line + '\n' ) - fh.close() + open(submit_file, "w").write(submit_file_contents) except: if self.app.config.cleanup_job == "always": cjs.cleanup() @@ -132,23 +113,9 @@ log.debug( "(%s) submitting file %s" % ( galaxy_id_tag, executable ) ) log.debug( "(%s) command is: %s" % ( galaxy_id_tag, command_line ) ) - s_out = '' - external_job_id = None - try: - submit = subprocess.Popen( ( 'condor_submit', submit_file ), stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - s_out, s_err = submit.communicate() - if submit.returncode == 0: - match = re.search( 'submitted to cluster (\\d+).', s_out ) - if match is None: - s_out = 'Failed to find job id from condor_submit' - else: - external_job_id = match.group( 1 ) - except Exception, e: - # TODO Add extra except for OSError? - s_out = str(e) - + external_job_id, message = condor_submit(submit_file) if external_job_id is None: - log.debug( "condor_submit failed for job %s: %s" % (job_wrapper.get_id_tag(), s_out) ) + log.debug( "condor_submit failed for job %s: %s" % (job_wrapper.get_id_tag(), message) ) if self.app.config.cleanup_job == "always": os.unlink( submit_file ) cjs.cleanup() @@ -177,29 +144,17 @@ new_watched = [] for cjs in self.watched: job_id = cjs.job_id - log_job_id = job_id.zfill(3) galaxy_id_tag = cjs.job_wrapper.get_id_tag() - job_running = False - job_complete = False - job_failed = False try: if os.stat( cjs.user_log ).st_size == cjs.user_log_size: new_watched.append( cjs ) continue - with open(cjs.user_log, 'r') as fh: - for line in fh: - if '001 (' + log_job_id + '.' in line: - job_running = True - if '004 (' + log_job_id + '.' in line: - job_running = False - if '007 (' + log_job_id + '.' in line: - job_running = False - if '005 (' + log_job_id + '.' in line: - job_complete = True - if '009 (' + log_job_id + '.' in line: - job_failed = True - cjs.user_log_size = fh.tell() - except Exception, e: + s1, s4, s7, s5, s9, log_size = summarize_condor_log(cjs.user_log, job_id) + job_running = s1 and not (s4 or s7) + job_complete = s5 + job_failed = s9 + cjs.user_log_size = log_size + except Exception: # so we don't kill the monitor thread log.exception( "(%s/%s) Unable to check job status" % ( galaxy_id_tag, job_id ) ) log.warning( "(%s/%s) job will now be errored" % ( galaxy_id_tag, job_id ) ) @@ -230,13 +185,10 @@ def stop_job( self, job ): """Attempts to delete a job from the DRM queue""" - try: - subprocess.check_call( ( 'condor_rm', job.job_runner_external_id ) ) - log.debug( "(%s/%s) Removed from DRM queue at user's request" % ( job.id, job.job_runner_external_id ) ) - except subprocess.CalledProcessError: - log.debug( "(%s/%s) User killed running job, but condor_rm failed" % ( job.id, job.job_runner_external_id ) ) - except Exception, e: - log.debug( "(%s/%s) User killed running job, but error encountered removing from Condor queue: %s" % ( job.id, job.job_runner_external_id, e ) ) + external_id = job.job_runner_external_id + failure_message = condor_stop(external_id) + if failure_message: + log.debug("(%s/%s). Failed to stop condor %s" % (external_id, failure_message)) def recover( self, job, job_wrapper ): """Recovers jobs stuck in the queued/running state when Galaxy started""" diff -r 432999eabbaa97528a2bd37f1b99de86b39a924f -r 126d5c5b9a4938909b518cb176dc47890e1c13e5 lib/galaxy/jobs/runners/util/condor/__init__.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/condor/__init__.py @@ -0,0 +1,114 @@ +""" +Condor helper utilities. +""" +from subprocess import Popen, PIPE, STDOUT, check_call, CalledProcessError +from ..external import parse_external_id + +DEFAULT_QUERY_CLASSAD = dict( + universe='vanilla', + getenv='true', + notification='NEVER', +) + +PROBLEM_RUNNING_CONDOR_SUBMIT = \ + "Problem encountered while running condor_submit." +PROBLEM_PARSING_EXTERNAL_ID = \ + "Failed to find job id from condor_submit" + +SUBMIT_PARAM_PREFIX = "submit_" + + +def submission_params(**kwds): + submission_params = {} + for key, value in kwds.iteritems(): + key = key.lower() + if key.startswith(SUBMIT_PARAM_PREFIX): + condor_key = key[len(SUBMIT_PARAM_PREFIX):] + submission_params[condor_key] = value + return submission_params + + +def build_submit_description(executable, output, error, user_log, query_params): + """ + Build up the contents of a condor submit description file. + + >>> submit_args = dict(executable='/path/to/script', output='o', error='e', user_log='ul') + >>> submit_args['query_params'] = dict() + >>> default_description = build_submit_description(**submit_args) + >>> assert 'executable = /path/to/script' in default_description + >>> assert 'output = o' in default_description + >>> assert 'error = e' in default_description + >>> assert 'queue' in default_description + >>> assert 'universe = vanilla' in default_description + >>> assert 'universe = standard' not in default_description + >>> submit_args['query_params'] = dict(universe='standard') + >>> std_description = build_submit_description(**submit_args) + >>> assert 'universe = vanilla' not in std_description + >>> assert 'universe = standard' in std_description + """ + all_query_params = DEFAULT_QUERY_CLASSAD.copy() + all_query_params.update(query_params) + + submit_description = [] + for key, value in all_query_params.items(): + submit_description.append('%s = %s' % (key, value)) + submit_description.append('executable = ' + executable) + submit_description.append('output = ' + output) + submit_description.append('error = ' + error) + submit_description.append('log = ' + user_log) + submit_description.append('queue') + return '\n'.join(submit_description) + + +def condor_submit(submit_file): + """ + Submit a condor job described by the given file. Parse an external id for + the submission or return None and a reason for the failure. + """ + external_id = None + try: + submit = Popen(('condor_submit', submit_file), stdout=PIPE, stderr=STDOUT) + message, _ = submit.communicate() + if submit.returncode == 0: + external_id = parse_external_id(message, type='condor') + else: + message = PROBLEM_PARSING_EXTERNAL_ID + except Exception as e: + message = str(e) + return external_id, message + + +def condor_stop(external_id): + """ + Stop running condor job and return a failure_message if this + fails. + """ + failure_message = None + try: + check_call(('condor_rm', external_id)) + except CalledProcessError: + failure_message = "condor_rm failed" + except Exception as e: + "error encountered calling condor_rm: %s" % e + return failure_message + + +def summarize_condor_log(log_file, external_id): + """ + """ + log_job_id = external_id.zfill(3) + s1 = s4 = s7 = s5 = s9 = False + with open(log_file, 'r') as log_handle: + for line in log_handle: + if '001 (' + log_job_id + '.' in line: + s1 = True + if '004 (' + log_job_id + '.' in line: + s4 = True + if '007 (' + log_job_id + '.' in line: + s7 = True + if '005 (' + log_job_id + '.' in line: + s5 = True + if '009 (' + log_job_id + '.' in line: + s9 = True + file_size = log_handle.tell() + return s1, s4, s7, s5, s9, file_size diff -r 432999eabbaa97528a2bd37f1b99de86b39a924f -r 126d5c5b9a4938909b518cb176dc47890e1c13e5 lib/galaxy/jobs/runners/util/external.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/external.py @@ -0,0 +1,37 @@ +from re import search + +EXTERNAL_ID_TYPE_ANY = None + +EXTERNAL_ID_PATTERNS = [ + ('condor', r'submitted to cluster (\d+)\.'), + ('slurm', r'Submitted batch job (\w+)'), + ('torque', r'(.+)'), # Default 'pattern' assumed by Galaxy code circa August 2013. +] + + +def parse_external_id(output, type=EXTERNAL_ID_TYPE_ANY): + """ + Attempt to parse the output of job submission commands for an external id.__doc__ + + >>> parse_external_id("12345.pbsmanager") + '12345.pbsmanager' + >>> parse_external_id('Submitted batch job 185') + '185' + >>> parse_external_id('Submitted batch job 185', type='torque') + 'Submitted batch job 185' + >>> parse_external_id('submitted to cluster 125.') + '125' + >>> parse_external_id('submitted to cluster 125.', type='slurm') + >>> + """ + external_id = None + for pattern_type, pattern in EXTERNAL_ID_PATTERNS: + if type != EXTERNAL_ID_TYPE_ANY and type != pattern_type: + continue + + match = search(pattern, output) + if match: + external_id = match.group(1) + break + + return external_id https://bitbucket.org/galaxy/galaxy-central/commits/93caaf7f9bdd/ Changeset: 93caaf7f9bdd User: jmchilton Date: 2013-10-17 00:37:37 Summary: Implement GALAXY_SLOTS logic for condor runner. Just pass through submit_request_cpus as GALAXY_SLOTS if set in job_conf.xml. Affected #: 1 file diff -r 126d5c5b9a4938909b518cb176dc47890e1c13e5 -r 93caaf7f9bdd6429a250921fa080bfe71028f8c0 lib/galaxy/jobs/runners/condor.py --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -58,6 +58,11 @@ # get destination params query_params = submission_params(**job_destination.params) + galaxy_slots = query_params.get('request_cpus', None) + if galaxy_slots: + galaxy_slots_statement = 'GALAXY_SLOTS="%s"; export GALAXY_SLOTS_CONFIGURED="1"' % galaxy_slots + else: + galaxy_slots_statement = 'GALAXY_SLOTS="1"' # define job attributes cjs = CondorJobState( @@ -80,7 +85,11 @@ ) submit_file_contents = build_submit_description(**build_submit_params) - script = self.get_job_file( job_wrapper, exit_code_path=cjs.exit_code_file ) + script = self.get_job_file( + job_wrapper, + exit_code_path=cjs.exit_code_file, + slots_statement=galaxy_slots_statement, + ) try: fh = file( executable, "w" ) fh.write( script ) https://bitbucket.org/galaxy/galaxy-central/commits/5715b876f91b/ Changeset: 5715b876f91b User: jmchilton Date: 2013-10-17 00:37:37 Summary: PBS job runner improvements (including GALAXY_SLOTS). Rework to use get_job_file, this will enable GALAXY_SLOTS logic for the PBS runner. Bring in staging fixes found in pull request #194 from CSIRO (stephen_mcmahon_). Small PEP-8 tweaks. Affected #: 1 file diff -r 93caaf7f9bdd6429a250921fa080bfe71028f8c0 -r 5715b876f91bd34d179dc9d48e7ad1a9e6baf285 lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -1,9 +1,10 @@ -import os, logging, threading, time, traceback +import os +import logging +import time +import traceback from datetime import timedelta -from Queue import Queue, Empty from galaxy import model -from galaxy.datatypes.data import nice_size from galaxy.util.bunch import Bunch from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size from galaxy.jobs import JobDestination @@ -35,42 +36,14 @@ # The last two lines execute the command and then retrieve the command's # exit code ($?) and write it to a file. -pbs_template = """#!/bin/sh -GALAXY_LIB="%s" -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - export PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - export PYTHONPATH="$GALAXY_LIB" - fi -fi -%s -cd %s -%s -echo $? > %s -""" - -# The last two lines execute the command and then retrieve the command's -# exit code ($?) and write it to a file. -pbs_symlink_template = """#!/bin/sh -GALAXY_LIB="%s" -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - export PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - export PYTHONPATH="$GALAXY_LIB" - fi -fi +pbs_symlink_template = """ for dataset in %s; do dir=`dirname $dataset` file=`basename $dataset` [ ! -d $dir ] && mkdir -p $dir [ ! -e $dataset ] && ln -s %s/$file $dataset done -%s -cd %s -%s -echo $? > %s +mkdir -p %s """ PBS_ARGMAP = { @@ -109,6 +82,7 @@ -8: "exec() of user command failed", } + class PBSJobRunner( AsynchronousJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling @@ -273,6 +247,7 @@ pbs_ofile = self.app.config.pbs_application_server + ':' + ofile pbs_efile = self.app.config.pbs_application_server + ':' + efile output_files = [ str( o ) for o in output_fnames ] + output_files.append(ecfile) stagein = self.get_stage_in_out( job_wrapper.get_input_fnames() + output_files, symlink=True ) stageout = self.get_stage_in_out( output_files ) attrs = [ @@ -300,20 +275,20 @@ # write the job script if self.app.config.pbs_stage_path != '': - script = pbs_symlink_template % ( job_wrapper.galaxy_lib_dir, - " ".join( job_wrapper.get_input_fnames() + output_files ), - self.app.config.pbs_stage_path, - job_wrapper.get_env_setup_clause(), - exec_dir, - command_line, - ecfile ) + # touch the ecfile so that it gets staged + with file(ecfile, 'a'): + os.utime(ecfile, None) + stage_commands = pbs_symlink_template % ( + " ".join( job_wrapper.get_input_fnames() + output_files ), + self.app.config.pbs_stage_path, + exec_dir, + ) else: - script = pbs_template % ( job_wrapper.galaxy_lib_dir, - job_wrapper.get_env_setup_clause(), - exec_dir, - command_line, - ecfile ) + stage_commands = '' + + env_setup_commands = '%s\n%s' % (stage_commands, job_wrapper.get_env_setup_clause()) + script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands) job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) fh = file(job_file, "w") fh.write(script) https://bitbucket.org/galaxy/galaxy-central/commits/84784515c048/ Changeset: 84784515c048 User: jmchilton Date: 2013-10-17 00:37:37 Summary: Add GALAXY_SLOTS logic to local runner. Affected #: 1 file diff -r 5715b876f91bd34d179dc9d48e7ad1a9e6baf285 -r 84784515c0481cfddbfc05e983efced55972e9a0 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -42,6 +42,21 @@ super( LocalJobRunner, self ).__init__( app, nworkers ) self._init_worker_threads() + def __command_line( self, job_wrapper ): + """ + """ + command_line = job_wrapper.runner_command_line + + ## slots would be cleaner name, but don't want deployers to see examples and think it + ## is going to work with other job runners. + slots = job_wrapper.job_destination.params.get( "local_slots", None ) + command_line = command_line.lstrip( " ;" ) + if slots: + command_line = 'export GALAXY_SLOTS="%d"; export GALAXY_SLOTS_CONFIGURED="1"; %s' % ( int( slots ), command_line ) + else: + command_line = 'export GALAXY_SLOTS="1"; %s' % command_line + return command_line + def queue_job( self, job_wrapper ): # prepare the job if not self.prepare_job( job_wrapper ): @@ -51,7 +66,7 @@ exit_code = 0 # command line has been added to the wrapper by prepare_job() - command_line = job_wrapper.runner_command_line + command_line = self.__command_line( job_wrapper ) job_id = job_wrapper.get_id_tag() https://bitbucket.org/galaxy/galaxy-central/commits/3825c8d49364/ Changeset: 3825c8d49364 User: jmchilton Date: 2013-10-17 00:37:37 Summary: Improved fix for tools producing command lines with trailing semi-colons. Affected #: 1 file diff -r 84784515c0481cfddbfc05e983efced55972e9a0 -r 3825c8d493649a72c6fc65bdd06b2362f0bf5930 lib/galaxy/jobs/command_factory.py --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -21,7 +21,7 @@ # Remove trailing semi-colon so we can start hacking up this command. # TODO: Refactor to compose a list and join with ';', would be more clean. - commands = commands.rstrip(";") + commands = commands.rstrip("; ") # Prepend version string if job_wrapper.version_string_cmd: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the Tool Shed API feature supporting resetting metadata on certain repositories to allow for individual repositories per request.
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b681b2aeddbc/ Changeset: b681b2aeddbc User: greg Date: 2013-10-16 22:29:00 Summary: Enhance the Tool Shed API feature supporting resetting metadata on certain repositories to allow for individual repositories per request. Affected #: 3 files diff -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a -r b681b2aeddbc52c3d953abcb431688df6fb8a005 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -177,6 +177,42 @@ return message @web.expose_api + def repository_ids_for_setting_metadata( self, trans, my_writable=False, **kwd ): + """ + GET /api/get_repository_ids_for_setting_metadata + + Displays a collection (list) of repository ids ordered for setting metadata. + + :param key: the API key of the Tool Shed user. + :param my_writable (optional): if the API key is associated with an admin user in the Tool Shed, setting this param value + to True will restrict resetting metadata to only repositories that are writable by the user + in addition to those repositories of type tool_dependency_definition. This param is ignored + if the current user is not an admin user, in which case this same restriction is automatic. + """ + try: + if trans.user_is_admin(): + my_writable = util.asbool( my_writable ) + else: + my_writable = True + handled_repository_ids = [] + repository_ids = [] + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # Make sure repositories of type tool_dependency_definition are first in the list. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + # Now add all remaining repositories to the list. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + return repository_ids + except Exception, e: + message = "Error in the Tool Shed repositories API in repository_ids_for_setting_metadata: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + + @web.expose_api def reset_metadata_on_repositories( self, trans, payload, **kwd ): """ PUT /api/repositories/reset_metadata_on_repositories @@ -239,6 +275,48 @@ trans.response.status = 500 return message + @web.expose_api + def reset_metadata_on_repository( self, trans, payload, **kwd ): + """ + PUT /api/repositories/reset_metadata_on_repository + + Resets all metadata on a specified repository in the Tool Shed. + + :param key: the API key of the Tool Shed user. + + The following parameters must be included in the payload. + :param repository_id: the encoded id of the repository on which metadata is to be reset. + """ + def handle_repository( trans, start_time, repository ): + results = dict( start_time=start_time, + repository_status=[] ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, + trans.security.encode_id( repository.id ) ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + + results[ 'repository_status' ].append( message ) + return results + try: + repository_id = payload.get( 'repository_id', None ) + if repository_id is not None: + repository = suc.get_repository_in_tool_shed( trans, repository_id ) + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = handle_repository( trans, start_time, repository ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ diff -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a -r b681b2aeddbc52c3d953abcb431688df6fb8a005 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -85,7 +85,9 @@ controller='repositories', collection={ 'get_repository_revision_install_info' : 'GET', 'get_ordered_installable_revisions' : 'GET', - 'reset_metadata_on_repositories' : 'POST' }, + 'repository_ids_for_setting_metadata' : 'GET', + 'reset_metadata_on_repositories' : 'POST', + 'reset_metadata_on_repository' : 'POST' }, name_prefix='repository_', path_prefix='/api', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) diff -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a -r b681b2aeddbc52c3d953abcb431688df6fb8a005 lib/tool_shed/scripts/api/reset_metadata_on_repositories.py --- a/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py +++ b/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py @@ -16,20 +16,37 @@ import os import sys sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import get from common import submit +def string_as_bool( string ): + if str( string ).lower() in [ 'true' ]: + return True + else: + return False + def main( options ): api_key = options.api my_writable = options.my_writable + one_per_request = options.one_per_request base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) - data = dict( my_writable=my_writable ) - url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url - submit( url, data, options.api ) + if string_as_bool( one_per_request ): + url = '%s/api/repositories/repository_ids_for_setting_metadata?key=%s&my_writable=%s' % ( base_tool_shed_url, api_key, str( my_writable ) ) + repository_ids = get( url, api_key ) + for repository_id in repository_ids: + data = dict( repository_id=repository_id ) + url = '%s/api/repositories/reset_metadata_on_repository' % base_tool_shed_url + submit( url, data, options.api ) + else: + data = dict( my_writable=my_writable ) + url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url + submit( url, data, options.api ) if __name__ == '__main__': parser = argparse.ArgumentParser( description='Reset metadata on certain repositories in the Tool Shed via the Tool Shed API.' ) parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) parser.add_argument( "-m", "--my_writable", dest="my_writable", required=False, default='False', help="Restrict to my writable repositories" ) + parser.add_argument( "-o", "--one_per_request", dest="one_per_request", required=False, default='False', help="One repository per request" ) parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) options = parser.parse_args() main( options ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Allow user field (__user_email__, __user_name__, etc..) in hidden fields as outlined in PR 206, but do so in a way that centralizes some related logic (i.e. logic used by Cheetah in job templates and used by dynamic filter options).
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2a4f4be3c377/ Changeset: 2a4f4be3c377 User: jmchilton Date: 2013-09-19 21:46:45 Summary: Allow user field (__user_email__, __user_name__, etc..) in hidden fields as outlined in PR 206, but do so in a way that centralizes some related logic (i.e. logic used by Cheetah in job templates and used by dynamic filter options). Affected #: 4 files diff -r 64f283100f960c1ae7645fae17394334b1b7f349 -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -680,20 +680,8 @@ out_data[ "output_file" ] = FakeDatasetAssociation( dataset=special.dataset ) # These can be passed on the command line if wanted as $__user_*__ - if job.history and job.history.user: - user = job.history.user - user_id = '%d' % job.history.user.id - user_email = str(job.history.user.email) - user_name = str(job.history.user.username) - else: - user = None - user_id = 'Anonymous' - user_email = 'Anonymous' - user_name = 'Anonymous' - incoming['__user__'] = user - incoming['__user_id__'] = incoming['userId'] = user_id - incoming['__user_email__'] = incoming['userEmail'] = user_email - incoming['__user_name__'] = user_name + incoming.update( model.User.user_template_environment( job.history and job.history.user ) ) + # Build params, done before hook so hook can use param_dict = self.tool.build_param_dict( incoming, inp_data, out_data, diff -r 64f283100f960c1ae7645fae17394334b1b7f349 -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -18,6 +18,7 @@ import simplejson import socket import time +from string import Template import galaxy.datatypes import galaxy.datatypes.registry @@ -159,6 +160,47 @@ total += hda.dataset.get_total_size() return total + @staticmethod + def user_template_environment( user ): + """ + + >>> env = User.user_template_environment(None) + >>> env['__user_email__'] + 'Anonymous' + >>> env['__user_id__'] + 'Anonymous' + >>> user = User('foo(a)example.com') + >>> user.id = 6 + >>> user.username = 'foo2' + >>> env = User.user_template_environment(user) + >>> env['__user_id__'] + '6' + >>> env['__user_name__'] + 'foo2' + """ + if user: + user_id = '%d' % user.id + user_email = str( user.email ) + user_name = str( user.username ) + else: + user = None + user_id = 'Anonymous' + user_email = 'Anonymous' + user_name = 'Anonymous' + environment = {} + environment[ '__user__' ] = user + environment[ '__user_id__' ] = environment[ 'userId' ] = user_id + environment[ '__user_email__' ] = environment[ 'userEmail' ] = user_email + environment[ '__user_name__' ] = user_name + return environment + + @staticmethod + def expand_user_properties( user, in_string ): + """ + """ + environment = User.user_template_environment( user ) + return Template( in_string ).safe_substitute( environment ) + class Job( object, Dictifiable ): dict_collection_visible_keys = [ 'id' ] diff -r 64f283100f960c1ae7645fae17394334b1b7f349 -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2291,7 +2291,7 @@ args = dict() for key, param in self.inputs.iteritems(): if isinstance( param, HiddenToolParameter ): - args[key] = param.value + args[key] = model.User.expand_user_properties( trans.user, param.value ) elif isinstance( param, BaseURLToolParameter ): args[key] = param.get_value( trans ) else: diff -r 64f283100f960c1ae7645fae17394334b1b7f349 -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a lib/galaxy/tools/parameters/dynamic_options.py --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -6,6 +6,7 @@ import operator, sys, os, logging import basic, validation from galaxy.util import string_as_bool +from galaxy.model import User import galaxy.tools log = logging.getLogger(__name__) @@ -55,8 +56,7 @@ rval = [] filter_value = self.value try: - if trans.user.email: - filter_value = filter_value.replace('$__user_email__',trans.user.email) + filter_value = User.expand_user_properties( trans.user, filter_value) except: pass for fields in options: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the Tool Shed and Galaxy APIs to support resetting metadata on repositories in the Tool Shed or installed into Galaxy respectively. When resetting repository metadata in the Tool Shed, if the API key is associated with an admin user, then setting the my_writable param value to True will restrict resetting metadata to only repositories that are writable by the user in addition to those repositories of type tool_dependency_definition. This param is ignored
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/64f283100f96/ Changeset: 64f283100f96 User: greg Date: 2013-10-16 20:19:26 Summary: Enhance the Tool Shed and Galaxy APIs to support resetting metadata on repositories in the Tool Shed or installed into Galaxy respectively. When resetting repository metadata in the Tool Shed, if the API key is associated with an admin user, then setting the my_writable param value to True will restrict resetting metadata to only repositories that are writable by the user in addition to those repositories of type tool_dependency_definition. This param is ignored if the current user is not an admin user, in which case this same restriction is automatic. Also, when resetting metadata in the Tool Shed, repositories of type tool_dependency_definition are reset before repositories of type unrestricted. When setting metadata for installed repositories in Galaxy, neither repository types nor order are considered since these concepts are irrelevant in that environment. Affected #: 10 files diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -1,6 +1,7 @@ import logging from paste.httpexceptions import HTTPBadRequest, HTTPForbidden +from time import strftime from galaxy import util from galaxy import web @@ -10,6 +11,7 @@ from tool_shed.galaxy_install import repository_util from tool_shed.util import common_util from tool_shed.util import encoding_util +from tool_shed.util import metadata_util from tool_shed.util import workflow_util import tool_shed.util.shed_util_common as suc @@ -488,6 +490,49 @@ return tool_shed_repositories @web.expose_api + def reset_metadata_on_installed_repositories( self, trans, payload, **kwd ): + """ + PUT /api/tool_shed_repositories/reset_metadata_on_installed_repositories + + Resets all metadata on all repositories installed into Galaxy in an "orderly fashion". + + :param key: the API key of the Galaxy admin user. + """ + try: + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + successful_count=0, + unsuccessful_count=0, + repository_status=[] ) + # Make sure the current user's API key proves he is an admin user in this Galaxy instance. + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + # Now reset metadata on all remaining repositories. + for repository in query: + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + results[ 'unsuccessful_count' ] += 1 + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Galaxy tool_shed_repositories API in reset_metadata_on_installed_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + + @web.expose_api def show( self, trans, id, **kwd ): """ GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id} diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -187,6 +187,7 @@ 'exported_workflows' : 'GET', 'import_workflow' : 'POST', 'import_workflows' : 'POST' }, + collection={ 'reset_metadata_on_installed_repositories' : 'POST' }, controller='tool_shed_repositories', name_prefix='tool_shed_repository_', path_prefix='/api', diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -1,11 +1,16 @@ import logging +from time import strftime from galaxy.web.framework.helpers import time_ago from galaxy import eggs from galaxy import web from galaxy import util +from galaxy.util import json from galaxy.web.base.controller import BaseAPIController +import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc from tool_shed.galaxy_install import repository_util +from tool_shed.util import metadata_util +from tool_shed.util import tool_util eggs.require( 'mercurial' ) @@ -171,6 +176,69 @@ trans.response.status = 500 return message + @web.expose_api + def reset_metadata_on_repositories( self, trans, payload, **kwd ): + """ + PUT /api/repositories/reset_metadata_on_repositories + + Resets all metadata on all repositories in the Tool Shed in an "orderly fashion". Since there are currently only two + repository types (tool_dependecy_definition and unrestricted), the order in which metadata is reset is repositories of + type tool_dependecy_definition first followed by repositories of type unrestricted, and only one pass is necessary. If + a new repository type is introduced, the process will undoubtedly need to be revisited. To facilitate this order, an + in-memory list of repository ids that have been processed is maintained. + + :param key: the API key of the Tool Shed user. + + The following parameters can optionally be included in the payload. + :param my_writable (optional): if the API key is associated with an admin user in the Tool Shed, setting this param value + to True will restrict resetting metadata to only repositories that are writable by the user + in addition to those repositories of type tool_dependency_definition. This param is ignored + if the current user is not an admin user, in which case this same restriction is automatic. + """ + def handle_repository( trans, repository, results ): + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + results[ 'unsuccessful_count' ] += 1 + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + return results + try: + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + repository_status=[], + successful_count=0, + unsuccessful_count=0 ) + handled_repository_ids = [] + if trans.user_is_admin(): + my_writable = util.asbool( payload.get( 'my_writable', False ) ) + else: + my_writable = True + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # First reset metadata on all repositories of type repository_dependency_definition. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + # Now reset metadata on all remaining repositories. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -84,7 +84,8 @@ 'repositories', controller='repositories', collection={ 'get_repository_revision_install_info' : 'GET', - 'get_ordered_installable_revisions' : 'GET' }, + 'get_ordered_installable_revisions' : 'GET', + 'reset_metadata_on_repositories' : 'POST' }, name_prefix='repository_', path_prefix='/api', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/scripts/api/reset_metadata_on_repositories.py --- /dev/null +++ b/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Script to reset metadata on certain repositories in the Tool Shed. If the received API key is associated +with an admin user in the Tool Shed, setting the my_writable param value to True will restrict resetting +metadata to only repositories that are writable by the user in addition to those repositories of type +tool_dependency_definition. The my_writable param is ignored if the current user is not an admin user, +in which case this same restriction is automatic. + +usage: reset_metadata_on_repositories.py key <my_writable> + +Here is a working example of how to use this script to reset metadata on certain repositories in a specified Tool Shed. +python ./reset_metadata_on_repositories.py -a 22be3b -m True -u http://localhost:9009/ +""" + +import argparse +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api + my_writable = options.my_writable + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + data = dict( my_writable=my_writable ) + url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url + submit( url, data, options.api ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Reset metadata on certain repositories in the Tool Shed via the Tool Shed API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-m", "--my_writable", dest="my_writable", required=False, default='False', help="Restrict to my writable repositories" ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + options = parser.parse_args() + main( options ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1782,10 +1782,10 @@ log.debug( message ) unsuccessful_count += 1 else: - log.debug( "Successfully reset metadata on repository %s" % repository.name ) + log.debug( "Successfully reset metadata on repository %s" % str( repository.name ) ) successful_count += 1 except: - log.exception( "Error attempting to reset metadata on repository %s", repository.name ) + log.exception( "Error attempting to reset metadata on repository %s", str( repository.name ) ) unsuccessful_count += 1 message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) if unsuccessful_count: diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -20,6 +20,7 @@ from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import xml_util +import tool_shed.repository_types.util as rt_util from xml.etree import ElementTree as XmlET from urllib2 import HTTPError @@ -104,45 +105,15 @@ def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes', my_writable=False ): """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - if trans.webapp.name == 'tool_shed': - # We're in the tool shed. - if my_writable: - username = trans.user.username - clause_list = [] - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ): - allow_push = repository.allow_push( trans.app ) - if allow_push: - allow_push_usernames = allow_push.split( ',' ) - if username in allow_push_usernames: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - if clause_list: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( or_( *clause_list ) ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) + query = get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=True ) + for repository in query: + if trans.webapp.name == 'tool_shed': + owner = str( repository.user.username ) else: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - else: - # We're in Galaxy. - for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ - .order_by( trans.model.ToolShedRepository.table.c.name, - trans.model.ToolShedRepository.table.c.owner ): - option_label = '%s (%s)' % ( repository.name, repository.owner ) - option_value = trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) + owner = str( repository.owner ) + option_label = '%s (%s)' % ( str( repository.name ), owner ) + option_value = '%s' % trans.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) return repositories_select_field def build_tool_dependencies_select_field( trans, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ): @@ -836,6 +807,63 @@ prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict +def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=True ): + """ + Return a query containing repositories for resetting metadata. This method is called from both the Tool Shed and Galaxy. The + my_writable parameter is ignored unless called from the Tool Shed, and the order parameter is used for displaying the list of + repositories ordered alphabetically for display on a page. When called from wither the Tool Shed or Galaxy API, order is False. + """ + if trans.webapp.name == 'tool_shed': + # When called from the Tool Shed API, the metadata is reset on all repositories of type tool_dependency_definition in addition + # to other selected repositories. + if my_writable: + username = trans.user.username + clause_list = [] + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ): + allow_push = repository.allow_push( trans.app ) + if not order: + # We've been called from the Tool Shed API, so reset metadata on all repositories of type tool_dependency_definition. + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + elif allow_push: + # Include all repositories that are writable by the current user. + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + if clause_list: + if order: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( or_( *clause_list ) ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ) + else: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( or_( *clause_list ) ) + else: + # Return an empty query. + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.id == -1 ) + else: + if order: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ) + else: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) + else: + # We're in Galaxy. + if order: + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ + .order_by( trans.model.ToolShedRepository.table.c.name, + trans.model.ToolShedRepository.table.c.owner ) + else: + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) + def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 scripts/api/import_workflows_from_installed_tool_shed_repository.py --- a/scripts/api/import_workflows_from_installed_tool_shed_repository.py +++ b/scripts/api/import_workflows_from_installed_tool_shed_repository.py @@ -45,7 +45,7 @@ data = {} # NOTE: to import a single workflow, add an index to data (e.g., # data[ 'index' ] = 0 - # and change the url to be ~/import_workflow (simgular). For example, + # and change the url to be ~/import_workflow (singular). For example, # url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflow' % str( tool_shed_repository_id ) ) url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflows' % str( tool_shed_repository_id ) ) submit( options.api, url, data ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 scripts/api/reset_metadata_on_installed_repositories.py --- /dev/null +++ b/scripts/api/reset_metadata_on_installed_repositories.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Script to reset metadata on all Tool Shed repositories installed into a Galaxy instance. The received +API key must be associated with a Galaxy admin user. + +usage: reset_metadata_on_installed_repositories.py key + +Here is a working example of how to use this script. +python ./reset_metadata_on_installed_repositories.py -a 22be3b -u http://localhost:8763/ +""" + +import argparse +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api + base_galaxy_url = options.galaxy_url.rstrip( '/' ) + url = '%s/api/tool_shed_repositories/reset_metadata_on_installed_repositories' % base_galaxy_url + submit( options.api, url, {} ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Reset metadata on all Tool Shed repositories installed into Galaxy via the Galaxy API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-u", "--url", dest="galaxy_url", required=True, help="Galaxy URL" ) + options = parser.parse_args() + main( options ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -969,7 +969,7 @@ self.check_for_strings( [ 'Metadata has been reset' ] ) def reset_metadata_on_selected_repositories( self, repository_ids ): - self.visit_url( '/admin/reset_metadata_on_selected_repositories_in_tool_shed' ) + self.visit_url( '/admin/reset_metadata_on_selected_repositories' ) kwd = dict( repository_ids=repository_ids ) self.submit_form( form_no=1, button="reset_metadata_on_selected_repositories_button", **kwd ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e786022dc67e/ Changeset: e786022dc67e User: jmchilton Date: 2013-10-16 20:01:15 Summary: Improved handling of tools producing unicode in standard error/output. Without this change (modification on idea by Bjoern), sqlite reports errors as follows for such tools: ProgrammingError: (ProgrammingError) You must not use 8-bit bytestrings unless you use a text_factory that can interpret 8-bit bytestrings (like text_factory = str). It is highly recommended that you instead just switch your application to Unicode strings. u'UPDATE job SET update_time=?, stdout=?, stderr=? WHERE job.id = ?' ('2013-10-10 05:00:05.789309', '', '/dev/fd/2 unicode \xc3\xaa\xe0\xaf\xb3\xe0\xbe\x85\xe1\x9d\xb1\xe3\x8e\xb0\n', 11). Affected #: 1 file diff -r 1c22e43fc21e049da34af60f603f2e19bde912d2 -r e786022dc67ed918050bd81b9ac679ac958e4f75 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -25,6 +25,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext from galaxy.util.json import from_json_string +from galaxy.util import unicodify from .output_checker import check_output log = logging.getLogger( __name__ ) @@ -869,6 +870,9 @@ the output datasets based on stderr and stdout from the command, and the contents of the output files. """ + stdout = unicodify( stdout ) + stderr = unicodify( stderr ) + # default post job setup self.sa_session.expunge_all() job = self.get_job() @@ -1518,6 +1522,9 @@ the output datasets based on stderr and stdout from the command, and the contents of the output files. """ + stdout = unicodify( stdout ) + stderr = unicodify( stderr ) + # This may have ended too soon log.debug( 'task %s for job %d ended; exit code: %d' % (self.task_id, self.job_id, https://bitbucket.org/galaxy/galaxy-central/commits/00764f2c46df/ Changeset: 00764f2c46df User: jmchilton Date: 2013-10-16 20:01:15 Summary: Improved handling of tools producing unicode in stardard error/output. Without this modification paste cannot render the stderr and/or stdout of such tools. This is a modification on an idea from Simon Guest, utilizing some code (now available as galaxy.util.smart_str) from an older version of Django to solve this problem. Affected #: 2 files diff -r e786022dc67ed918050bd81b9ac679ac958e4f75 -r 00764f2c46dff45f208e22203c6498777725bba7 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -547,6 +547,30 @@ except: return default + +def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Returns a bytestring version of 's', encoded as specified in 'encoding'. + + If strings_only is True, don't convert (some) non-string-like objects. + + Adapted from an older, simpler version of django.utils.encoding.smart_str. + """ + if strings_only and isinstance(s, (type(None), int)): + return s + if not isinstance(s, basestring): + try: + return str(s) + except UnicodeEncodeError: + return unicode(s).encode(encoding, errors) + elif isinstance(s, unicode): + return s.encode(encoding, errors) + elif s and encoding != 'utf-8': + return s.decode('utf-8', errors).encode(encoding, errors) + else: + return s + + def object_to_string( obj ): return binascii.hexlify( pickle.dumps( obj, 2 ) ) diff -r e786022dc67ed918050bd81b9ac679ac958e4f75 -r 00764f2c46dff45f208e22203c6498777725bba7 lib/galaxy/webapps/galaxy/controllers/dataset.py --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -8,7 +8,7 @@ from galaxy import datatypes, eggs, model, util, web from galaxy.datatypes.display_applications.util import decode_dataset_user, encode_dataset_user from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings -from galaxy.util import inflector +from galaxy.util import inflector, smart_str from galaxy.util.sanitize_html import sanitize_html from galaxy.web.base.controller import BaseUIController, ERROR, SUCCESS, url_for, UsesHistoryDatasetAssociationMixin, UsesHistoryMixin from galaxy.web.framework.helpers import grids, iff, time_ago @@ -184,7 +184,7 @@ stdout = job.stdout except: stdout = "Invalid dataset ID or you are not allowed to access this dataset" - return stdout + return smart_str( stdout ) @web.expose # TODO: Migrate stderr and stdout to use _get_job_for_dataset; it wasn't tested. @@ -196,7 +196,7 @@ stderr = job.stderr except: stderr = "Invalid dataset ID or you are not allowed to access this dataset" - return stderr + return smart_str( stderr ) @web.expose def exit_code( self, trans, dataset_id=None, **kwargs ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3a156f9e008d/ Changeset: 3a156f9e008d User: nsoranzo Date: 2013-09-18 14:43:59 Summary: Prevent no choice in multiple selections. Affected #: 2 files diff -r 750e687b468a27978d51dfac9d542a0cc113e64b -r 3a156f9e008d3e61a951aa9efba2c982cb826e0a tools/ngs_simulation/ngs_simulation.py --- a/tools/ngs_simulation/ngs_simulation.py +++ b/tools/ngs_simulation/ngs_simulation.py @@ -70,11 +70,11 @@ num_sims = int( options.num_sims ) except TypeError, e: stop_err( 'Make sure the number of simulations is an integer value: %s' % str( e ) ) - if len( options.polymorphism ) > 0: + if options.polymorphism != 'None': polymorphisms = [ float( p ) for p in options.polymorphism.split( ',' ) ] else: stop_err( 'Select at least one polymorphism value to use' ) - if len( options.detection_thresh ) > 0: + if options.detection_thresh != 'None': detection_threshes = [ float( dt ) for dt in options.detection_thresh.split( ',' ) ] else: stop_err( 'Select at least one detection threshold to use' ) diff -r 750e687b468a27978d51dfac9d542a0cc113e64b -r 3a156f9e008d3e61a951aa9efba2c982cb826e0a tools/ngs_simulation/ngs_simulation.xml --- a/tools/ngs_simulation/ngs_simulation.xml +++ b/tools/ngs_simulation/ngs_simulation.xml @@ -72,6 +72,7 @@ <option value="0.8">0.8</option><option value="0.9">0.9</option><option value="1.0">1.0</option> + <validator type="no_options" message="You must select at least one value" /></param><param name="detection_thresh" type="select" multiple="true" label="Detection thresholds"><option value="0.001">0.001</option> @@ -102,6 +103,7 @@ <option value="0.8">0.8</option><option value="0.9">0.9</option><option value="1.0">1.0</option> + <validator type="no_options" message="You must select at least one value" /></param><param name="summary_out" type="boolean" truevalue="true" falsevalue="false" checked="true" label="Include a (text) summary file for all the simulations" /><!-- <param name="sim_results" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Output all tabular simulation results" help="Number of polymorphisms times number of detection thresholds"/> https://bitbucket.org/galaxy/galaxy-central/commits/ff4bbde97901/ Changeset: ff4bbde97901 User: nsoranzo Date: 2013-09-18 14:44:42 Summary: Fix reference to 'Fetch taxonomic representation' tool. Affected #: 1 file diff -r 3a156f9e008d3e61a951aa9efba2c982cb826e0a -r ff4bbde97901cc3dd3ab9c36432e2d17a655265a tools/taxonomy/t2ps_wrapper.xml --- a/tools/taxonomy/t2ps_wrapper.xml +++ b/tools/taxonomy/t2ps_wrapper.xml @@ -47,7 +47,7 @@ **What it does** -Given taxonomy representation (produced by *Taxonomy manipulation->Fetch Taxonomic Ranks* tool) this utility produces a graphical representations of phylogenetic tree in PDF format. +Given taxonomy representation (produced by *Fetch taxonomic representation* tool) this utility produces a graphical representations of phylogenetic tree in PDF format. -------- https://bitbucket.org/galaxy/galaxy-central/commits/c61cc2f91c59/ Changeset: c61cc2f91c59 User: nsoranzo Date: 2013-10-04 18:15:22 Summary: Fix for "syntax error near unexpected token `;;'". <command> should not end with a semicolon, otherwise BaseJobRunner.build_command_line() would create a bash command containing ';;' when appending metadata setting commands. Affected #: 1 file diff -r ff4bbde97901cc3dd3ab9c36432e2d17a655265a -r c61cc2f91c59c54e585ef92b77f650774ebdbd55 tools/sr_mapping/bowtie2_wrapper.xml --- a/tools/sr_mapping/bowtie2_wrapper.xml +++ b/tools/sr_mapping/bowtie2_wrapper.xml @@ -95,15 +95,15 @@ #end if ## view/sort and output file - | samtools view -Su - | samtools sort -o - - > $output; + | samtools view -Su - | samtools sort -o - - > $output ## rename unaligned sequence files #if $library.type == "paired" and $output_unaligned_reads_l and $output_unaligned_reads_r: #set left = str($output_unaligned_reads_l).replace( '.dat', '.1.dat' ) #set right = str($output_unaligned_reads_l).replace( '.dat', '.2.dat' ) - mv $left $output_unaligned_reads_l; - mv $right $output_unaligned_reads_r; + ; mv $left $output_unaligned_reads_l; + mv $right $output_unaligned_reads_r #end if </command> https://bitbucket.org/galaxy/galaxy-central/commits/1c22e43fc21e/ Changeset: 1c22e43fc21e User: jmchilton Date: 2013-10-16 19:49:34 Summary: Merged in nsoranzo/galaxy-central-tools2 (pull request #234) Bug fixes for 3 tools (rebased #222) Affected #: 4 files diff -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be -r 1c22e43fc21e049da34af60f603f2e19bde912d2 tools/ngs_simulation/ngs_simulation.py --- a/tools/ngs_simulation/ngs_simulation.py +++ b/tools/ngs_simulation/ngs_simulation.py @@ -70,11 +70,11 @@ num_sims = int( options.num_sims ) except TypeError, e: stop_err( 'Make sure the number of simulations is an integer value: %s' % str( e ) ) - if len( options.polymorphism ) > 0: + if options.polymorphism != 'None': polymorphisms = [ float( p ) for p in options.polymorphism.split( ',' ) ] else: stop_err( 'Select at least one polymorphism value to use' ) - if len( options.detection_thresh ) > 0: + if options.detection_thresh != 'None': detection_threshes = [ float( dt ) for dt in options.detection_thresh.split( ',' ) ] else: stop_err( 'Select at least one detection threshold to use' ) diff -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be -r 1c22e43fc21e049da34af60f603f2e19bde912d2 tools/ngs_simulation/ngs_simulation.xml --- a/tools/ngs_simulation/ngs_simulation.xml +++ b/tools/ngs_simulation/ngs_simulation.xml @@ -72,6 +72,7 @@ <option value="0.8">0.8</option><option value="0.9">0.9</option><option value="1.0">1.0</option> + <validator type="no_options" message="You must select at least one value" /></param><param name="detection_thresh" type="select" multiple="true" label="Detection thresholds"><option value="0.001">0.001</option> @@ -102,6 +103,7 @@ <option value="0.8">0.8</option><option value="0.9">0.9</option><option value="1.0">1.0</option> + <validator type="no_options" message="You must select at least one value" /></param><param name="summary_out" type="boolean" truevalue="true" falsevalue="false" checked="true" label="Include a (text) summary file for all the simulations" /><!-- <param name="sim_results" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Output all tabular simulation results" help="Number of polymorphisms times number of detection thresholds"/> diff -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be -r 1c22e43fc21e049da34af60f603f2e19bde912d2 tools/sr_mapping/bowtie2_wrapper.xml --- a/tools/sr_mapping/bowtie2_wrapper.xml +++ b/tools/sr_mapping/bowtie2_wrapper.xml @@ -95,15 +95,15 @@ #end if ## view/sort and output file - | samtools view -Su - | samtools sort -o - - > $output; + | samtools view -Su - | samtools sort -o - - > $output ## rename unaligned sequence files #if $library.type == "paired" and $output_unaligned_reads_l and $output_unaligned_reads_r: #set left = str($output_unaligned_reads_l).replace( '.dat', '.1.dat' ) #set right = str($output_unaligned_reads_l).replace( '.dat', '.2.dat' ) - mv $left $output_unaligned_reads_l; - mv $right $output_unaligned_reads_r; + ; mv $left $output_unaligned_reads_l; + mv $right $output_unaligned_reads_r #end if </command> diff -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be -r 1c22e43fc21e049da34af60f603f2e19bde912d2 tools/taxonomy/t2ps_wrapper.xml --- a/tools/taxonomy/t2ps_wrapper.xml +++ b/tools/taxonomy/t2ps_wrapper.xml @@ -47,7 +47,7 @@ **What it does** -Given taxonomy representation (produced by *Taxonomy manipulation->Fetch Taxonomic Ranks* tool) this utility produces a graphical representations of phylogenetic tree in PDF format. +Given taxonomy representation (produced by *Fetch taxonomic representation* tool) this utility produces a graphical representations of phylogenetic tree in PDF format. -------- Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6fdf96b4aef0/ Changeset: 6fdf96b4aef0 User: jmchilton Date: 2013-10-11 17:40:43 Summary: Refactor building command lines out of job runner base into its own module. Add unit tests. Test skipping metadata, using metadata, return code patch, dependency shell commands. Affected #: 3 files diff -r 829c047e05776fb2a6568f0d1076856fe6255fab -r 6fdf96b4aef00174938470f945d6931746e319f2 lib/galaxy/jobs/command_factory.py --- /dev/null +++ b/lib/galaxy/jobs/command_factory.py @@ -0,0 +1,70 @@ +from os import getcwd +from os.path import abspath + + +def build_command( job, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): + """ + Compose the sequence of commands necessary to execute a job. This will + currently include: + + - environment settings corresponding to any requirement tags + - preparing input files + - command line taken from job wrapper + - commands to set metadata (if include_metadata is True) + """ + + commands = job_wrapper.get_command_line() + + # All job runners currently handle this case which should never occur + if not commands: + return None + + # Prepend version string + if job_wrapper.version_string_cmd: + commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands + + # prepend getting input files (if defined) + if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: + commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] ) + + # Prepend dependency injection + if job_wrapper.dependency_shell_commands: + commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) + + # Coping work dir outputs or setting metadata will mask return code of + # tool command. If these are used capture the return code and ensure + # the last thing that happens is an exit with return code. + capture_return_code_command = "; return_code=$?" + captured_return_code = False + + # Append commands to copy job outputs based on from_work_dir attribute. + if include_work_dir_outputs: + work_dir_outputs = job.get_work_dir_outputs( job_wrapper ) + if work_dir_outputs: + if not captured_return_code: + commands += capture_return_code_command + captured_return_code = True + + commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" % + ( source_file, source_file, destination ) for ( source_file, destination ) in work_dir_outputs ] ) + + # Append metadata setting commands, we don't want to overwrite metadata + # that was copied over in init_meta(), as per established behavior + if include_metadata and job_wrapper.requires_setting_metadata: + if not captured_return_code: + commands += capture_return_code_command + captured_return_code = True + commands += "; cd %s; " % abspath( getcwd() ) + commands += job_wrapper.setup_external_metadata( + exec_dir=abspath( getcwd() ), + tmp_dir=job_wrapper.working_directory, + dataset_files_path=job.app.model.Dataset.file_path, + output_fnames=job_wrapper.get_output_fnames(), + set_extension=False, + kwds={ 'overwrite' : False } + ) + + if captured_return_code: + commands += '; sh -c "exit $return_code"' + + return commands diff -r 829c047e05776fb2a6568f0d1076856fe6255fab -r 6fdf96b4aef00174938470f945d6931746e319f2 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -12,6 +12,7 @@ from Queue import Queue, Empty import galaxy.jobs +from galaxy.jobs.command_factory import build_command from galaxy import model from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size @@ -142,67 +143,7 @@ raise NotImplementedError() def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): - """ - Compose the sequence of commands necessary to execute a job. This will - currently include: - - - environment settings corresponding to any requirement tags - - preparing input files - - command line taken from job wrapper - - commands to set metadata (if include_metadata is True) - """ - - commands = job_wrapper.get_command_line() - # All job runners currently handle this case which should never - # occur - if not commands: - return None - # Prepend version string - if job_wrapper.version_string_cmd: - commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands - # prepend getting input files (if defined) - if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: - commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] ) - # Prepend dependency injection - if job_wrapper.dependency_shell_commands: - commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) - - # Coping work dir outputs or setting metadata will mask return code of - # tool command. If these are used capture the return code and ensure - # the last thing that happens is an exit with return code. - capture_return_code_command = "; return_code=$?" - captured_return_code = False - - # Append commands to copy job outputs based on from_work_dir attribute. - if include_work_dir_outputs: - work_dir_outputs = self.get_work_dir_outputs( job_wrapper ) - if work_dir_outputs: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True - commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" % - ( source_file, source_file, destination ) for ( source_file, destination ) in work_dir_outputs ] ) - - # Append metadata setting commands, we don't want to overwrite metadata - # that was copied over in init_meta(), as per established behavior - if include_metadata and job_wrapper.requires_setting_metadata: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True - commands += "; cd %s; " % os.path.abspath( os.getcwd() ) - commands += job_wrapper.setup_external_metadata( - exec_dir = os.path.abspath( os.getcwd() ), - tmp_dir = job_wrapper.working_directory, - dataset_files_path = self.app.model.Dataset.file_path, - output_fnames = job_wrapper.get_output_fnames(), - set_extension = False, - kwds = { 'overwrite' : False } ) - - - if captured_return_code: - commands += '; sh -c "exit $return_code"' - - return commands + return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs ) def get_work_dir_outputs( self, job_wrapper ): """ diff -r 829c047e05776fb2a6568f0d1076856fe6255fab -r 6fdf96b4aef00174938470f945d6931746e319f2 test/unit/test_command_factory.py --- /dev/null +++ b/test/unit/test_command_factory.py @@ -0,0 +1,79 @@ +from os import getcwd +from unittest import TestCase + +from galaxy.jobs.command_factory import build_command +from galaxy.util.bunch import Bunch + +MOCK_COMMAND_LINE = "/opt/galaxy/tools/bowtie /mnt/galaxyData/files/000/input000.dat" + + +class TestCommandFactory(TestCase): + + def setUp(self): + self.job_wrapper = MockJobWrapper() + self.job = Bunch(app=Bunch(model=Bunch(Dataset=Bunch(file_path="file_path")))) + self.include_metadata = False + self.include_work_dir_outputs = True + + def test_simplest_command(self): + self.include_work_dir_outputs = False + self.__assert_command_is( MOCK_COMMAND_LINE ) + + def test_shell_commands(self): + self.include_work_dir_outputs = False + dep_commands = [". /opt/galaxy/tools/bowtie/default/env.sh"] + self.job_wrapper.dependency_shell_commands = dep_commands + self.__assert_command_is( "%s; %s" % (dep_commands[0], MOCK_COMMAND_LINE) ) + + def test_set_metadata_skipped_if_unneeded(self): + self.include_metadata = True + self.include_work_dir_outputs = False + self.__assert_command_is( MOCK_COMMAND_LINE ) + + def test_set_metadata(self): + self.include_metadata = True + self.include_work_dir_outputs = False + metadata_line = "set_metadata_and_stuff.sh" + self.job_wrapper.metadata_line = metadata_line + expected_command = '%s; return_code=$?; cd %s; %s; sh -c "exit $return_code"' % (MOCK_COMMAND_LINE, getcwd(), metadata_line) + self.__assert_command_is( expected_command ) + + def __assert_command_is(self, expected_command): + command = self.__command() + self.assertEqual(command, expected_command) + + def __command(self): + kwds = dict( + job=self.job, + job_wrapper=self.job_wrapper, + include_metadata=self.include_metadata, + include_work_dir_outputs=self.include_work_dir_outputs, + ) + return build_command(**kwds) + + +class MockJobWrapper(object): + + def __init__(self): + self.version_string_cmd = None + self.command_line = MOCK_COMMAND_LINE + self.dependency_shell_commands = [] + self.metadata_line = None + self.working_directory = "job1" + + def get_command_line(self): + return self.command_line + + @property + def requires_setting_metadata(self): + return self.metadata_line is not None + + def setup_external_metadata(self, *args, **kwds): + return self.metadata_line + + def get_output_fnames(self): + return [] + + +class MockJob(object): + app = Bunch() https://bitbucket.org/galaxy/galaxy-central/commits/e0016057e164/ Changeset: e0016057e164 User: jmchilton Date: 2013-10-11 17:40:43 Summary: Fix bug related to trailing semi-colon in tools. Thanks to Bjoern, Nicola, Nate for helping track this down. Affected #: 2 files diff -r 6fdf96b4aef00174938470f945d6931746e319f2 -r e0016057e1648d7f4d29fbaf14252cbefff6d0bd lib/galaxy/jobs/command_factory.py --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -19,6 +19,10 @@ if not commands: return None + # Remove trailing semi-colon so we can start hacking up this command. + # TODO: Refactor to compose a list and join with ';', would be more clean. + commands = commands.rstrip(";") + # Prepend version string if job_wrapper.version_string_cmd: commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands diff -r 6fdf96b4aef00174938470f945d6931746e319f2 -r e0016057e1648d7f4d29fbaf14252cbefff6d0bd test/unit/test_command_factory.py --- a/test/unit/test_command_factory.py +++ b/test/unit/test_command_factory.py @@ -31,6 +31,13 @@ self.__assert_command_is( MOCK_COMMAND_LINE ) def test_set_metadata(self): + self._test_set_metadata() + + def test_strips_trailing_semicolons(self): + self.job_wrapper.command_line = "%s;" % MOCK_COMMAND_LINE + self._test_set_metadata() + + def _test_set_metadata(self): self.include_metadata = True self.include_work_dir_outputs = False metadata_line = "set_metadata_and_stuff.sh" https://bitbucket.org/galaxy/galaxy-central/commits/e8fbf32ba1cc/ Changeset: e8fbf32ba1cc User: jmchilton Date: 2013-10-16 19:48:50 Summary: Merge pull request #235. Fix for tools with trailing semi-colons. Affected #: 3 files diff -r e4d476ccf7832df0b1f65048d3784b010f84e59a -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be lib/galaxy/jobs/command_factory.py --- /dev/null +++ b/lib/galaxy/jobs/command_factory.py @@ -0,0 +1,74 @@ +from os import getcwd +from os.path import abspath + + +def build_command( job, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): + """ + Compose the sequence of commands necessary to execute a job. This will + currently include: + + - environment settings corresponding to any requirement tags + - preparing input files + - command line taken from job wrapper + - commands to set metadata (if include_metadata is True) + """ + + commands = job_wrapper.get_command_line() + + # All job runners currently handle this case which should never occur + if not commands: + return None + + # Remove trailing semi-colon so we can start hacking up this command. + # TODO: Refactor to compose a list and join with ';', would be more clean. + commands = commands.rstrip(";") + + # Prepend version string + if job_wrapper.version_string_cmd: + commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands + + # prepend getting input files (if defined) + if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: + commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] ) + + # Prepend dependency injection + if job_wrapper.dependency_shell_commands: + commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) + + # Coping work dir outputs or setting metadata will mask return code of + # tool command. If these are used capture the return code and ensure + # the last thing that happens is an exit with return code. + capture_return_code_command = "; return_code=$?" + captured_return_code = False + + # Append commands to copy job outputs based on from_work_dir attribute. + if include_work_dir_outputs: + work_dir_outputs = job.get_work_dir_outputs( job_wrapper ) + if work_dir_outputs: + if not captured_return_code: + commands += capture_return_code_command + captured_return_code = True + + commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" % + ( source_file, source_file, destination ) for ( source_file, destination ) in work_dir_outputs ] ) + + # Append metadata setting commands, we don't want to overwrite metadata + # that was copied over in init_meta(), as per established behavior + if include_metadata and job_wrapper.requires_setting_metadata: + if not captured_return_code: + commands += capture_return_code_command + captured_return_code = True + commands += "; cd %s; " % abspath( getcwd() ) + commands += job_wrapper.setup_external_metadata( + exec_dir=abspath( getcwd() ), + tmp_dir=job_wrapper.working_directory, + dataset_files_path=job.app.model.Dataset.file_path, + output_fnames=job_wrapper.get_output_fnames(), + set_extension=False, + kwds={ 'overwrite' : False } + ) + + if captured_return_code: + commands += '; sh -c "exit $return_code"' + + return commands diff -r e4d476ccf7832df0b1f65048d3784b010f84e59a -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -12,6 +12,7 @@ from Queue import Queue, Empty import galaxy.jobs +from galaxy.jobs.command_factory import build_command from galaxy import model from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size @@ -142,67 +143,7 @@ raise NotImplementedError() def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): - """ - Compose the sequence of commands necessary to execute a job. This will - currently include: - - - environment settings corresponding to any requirement tags - - preparing input files - - command line taken from job wrapper - - commands to set metadata (if include_metadata is True) - """ - - commands = job_wrapper.get_command_line() - # All job runners currently handle this case which should never - # occur - if not commands: - return None - # Prepend version string - if job_wrapper.version_string_cmd: - commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands - # prepend getting input files (if defined) - if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: - commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] ) - # Prepend dependency injection - if job_wrapper.dependency_shell_commands: - commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) - - # Coping work dir outputs or setting metadata will mask return code of - # tool command. If these are used capture the return code and ensure - # the last thing that happens is an exit with return code. - capture_return_code_command = "; return_code=$?" - captured_return_code = False - - # Append commands to copy job outputs based on from_work_dir attribute. - if include_work_dir_outputs: - work_dir_outputs = self.get_work_dir_outputs( job_wrapper ) - if work_dir_outputs: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True - commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" % - ( source_file, source_file, destination ) for ( source_file, destination ) in work_dir_outputs ] ) - - # Append metadata setting commands, we don't want to overwrite metadata - # that was copied over in init_meta(), as per established behavior - if include_metadata and job_wrapper.requires_setting_metadata: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True - commands += "; cd %s; " % os.path.abspath( os.getcwd() ) - commands += job_wrapper.setup_external_metadata( - exec_dir = os.path.abspath( os.getcwd() ), - tmp_dir = job_wrapper.working_directory, - dataset_files_path = self.app.model.Dataset.file_path, - output_fnames = job_wrapper.get_output_fnames(), - set_extension = False, - kwds = { 'overwrite' : False } ) - - - if captured_return_code: - commands += '; sh -c "exit $return_code"' - - return commands + return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs ) def get_work_dir_outputs( self, job_wrapper ): """ diff -r e4d476ccf7832df0b1f65048d3784b010f84e59a -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be test/unit/test_command_factory.py --- /dev/null +++ b/test/unit/test_command_factory.py @@ -0,0 +1,86 @@ +from os import getcwd +from unittest import TestCase + +from galaxy.jobs.command_factory import build_command +from galaxy.util.bunch import Bunch + +MOCK_COMMAND_LINE = "/opt/galaxy/tools/bowtie /mnt/galaxyData/files/000/input000.dat" + + +class TestCommandFactory(TestCase): + + def setUp(self): + self.job_wrapper = MockJobWrapper() + self.job = Bunch(app=Bunch(model=Bunch(Dataset=Bunch(file_path="file_path")))) + self.include_metadata = False + self.include_work_dir_outputs = True + + def test_simplest_command(self): + self.include_work_dir_outputs = False + self.__assert_command_is( MOCK_COMMAND_LINE ) + + def test_shell_commands(self): + self.include_work_dir_outputs = False + dep_commands = [". /opt/galaxy/tools/bowtie/default/env.sh"] + self.job_wrapper.dependency_shell_commands = dep_commands + self.__assert_command_is( "%s; %s" % (dep_commands[0], MOCK_COMMAND_LINE) ) + + def test_set_metadata_skipped_if_unneeded(self): + self.include_metadata = True + self.include_work_dir_outputs = False + self.__assert_command_is( MOCK_COMMAND_LINE ) + + def test_set_metadata(self): + self._test_set_metadata() + + def test_strips_trailing_semicolons(self): + self.job_wrapper.command_line = "%s;" % MOCK_COMMAND_LINE + self._test_set_metadata() + + def _test_set_metadata(self): + self.include_metadata = True + self.include_work_dir_outputs = False + metadata_line = "set_metadata_and_stuff.sh" + self.job_wrapper.metadata_line = metadata_line + expected_command = '%s; return_code=$?; cd %s; %s; sh -c "exit $return_code"' % (MOCK_COMMAND_LINE, getcwd(), metadata_line) + self.__assert_command_is( expected_command ) + + def __assert_command_is(self, expected_command): + command = self.__command() + self.assertEqual(command, expected_command) + + def __command(self): + kwds = dict( + job=self.job, + job_wrapper=self.job_wrapper, + include_metadata=self.include_metadata, + include_work_dir_outputs=self.include_work_dir_outputs, + ) + return build_command(**kwds) + + +class MockJobWrapper(object): + + def __init__(self): + self.version_string_cmd = None + self.command_line = MOCK_COMMAND_LINE + self.dependency_shell_commands = [] + self.metadata_line = None + self.working_directory = "job1" + + def get_command_line(self): + return self.command_line + + @property + def requires_setting_metadata(self): + return self.metadata_line is not None + + def setup_external_metadata(self, *args, **kwds): + return self.metadata_line + + def get_output_fnames(self): + return [] + + +class MockJob(object): + app = Bunch() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Restore improvements to job output checking code.
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e4d476ccf783/ Changeset: e4d476ccf783 User: jmchilton Date: 2013-10-16 19:46:59 Summary: Restore improvements to job output checking code. Affected #: 5 files diff -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e -r e4d476ccf7832df0b1f65048d3784b010f84e59a lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -25,6 +25,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext from galaxy.util.json import from_json_string +from .output_checker import check_output log = logging.getLogger( __name__ ) @@ -1079,158 +1080,7 @@ self.cleanup() def check_tool_output( self, stdout, stderr, tool_exit_code, job ): - """ - Check the output of a tool - given the stdout, stderr, and the tool's - exit code, return True if the tool exited succesfully and False - otherwise. No exceptions should be thrown. If this code encounters - an exception, it returns True so that the workflow can continue; - otherwise, a bug in this code could halt workflow progress. - Note that, if the tool did not define any exit code handling or - any stdio/stderr handling, then it reverts back to previous behavior: - if stderr contains anything, then False is returned. - Note that the job id is just for messages. - """ - # By default, the tool succeeded. This covers the case where the code - # has a bug but the tool was ok, and it lets a workflow continue. - success = True - - try: - # Check exit codes and match regular expressions against stdout and - # stderr if this tool was configured to do so. - # If there is a regular expression for scanning stdout/stderr, - # then we assume that the tool writer overwrote the default - # behavior of just setting an error if there is *anything* on - # stderr. - if ( len( self.tool.stdio_regexes ) > 0 or - len( self.tool.stdio_exit_codes ) > 0 ): - # Check the exit code ranges in the order in which - # they were specified. Each exit_code is a StdioExitCode - # that includes an applicable range. If the exit code was in - # that range, then apply the error level and add a message. - # If we've reached a fatal error rule, then stop. - max_error_level = galaxy.tools.StdioErrorLevel.NO_ERROR - if tool_exit_code != None: - for stdio_exit_code in self.tool.stdio_exit_codes: - if ( tool_exit_code >= stdio_exit_code.range_start and - tool_exit_code <= stdio_exit_code.range_end ): - # Tack on a generic description of the code - # plus a specific code description. For example, - # this might prepend "Job 42: Warning (Out of Memory)\n". - code_desc = stdio_exit_code.desc - if ( None == code_desc ): - code_desc = "" - tool_msg = ( "%s: Exit code %d (%s)" % ( - galaxy.tools.StdioErrorLevel.desc( stdio_exit_code.error_level ), - tool_exit_code, - code_desc ) ) - log.info( "Job %s: %s" % (job.get_id_tag(), tool_msg) ) - stderr = tool_msg + "\n" + stderr - max_error_level = max( max_error_level, - stdio_exit_code.error_level ) - if ( max_error_level >= - galaxy.tools.StdioErrorLevel.FATAL ): - break - - if max_error_level < galaxy.tools.StdioErrorLevel.FATAL: - # We'll examine every regex. Each regex specifies whether - # it is to be run on stdout, stderr, or both. (It is - # possible for neither stdout nor stderr to be scanned, - # but those regexes won't be used.) We record the highest - # error level, which are currently "warning" and "fatal". - # If fatal, then we set the job's state to ERROR. - # If warning, then we still set the job's state to OK - # but include a message. We'll do this if we haven't seen - # a fatal error yet - for regex in self.tool.stdio_regexes: - # If ( this regex should be matched against stdout ) - # - Run the regex's match pattern against stdout - # - If it matched, then determine the error level. - # o If it was fatal, then we're done - break. - # Repeat the stdout stuff for stderr. - # TODO: Collapse this into a single function. - if ( regex.stdout_match ): - regex_match = re.search( regex.match, stdout, - re.IGNORECASE ) - if ( regex_match ): - rexmsg = self.regex_err_msg( regex_match, regex) - log.info( "Job %s: %s" - % ( job.get_id_tag(), rexmsg ) ) - stdout = rexmsg + "\n" + stdout - max_error_level = max( max_error_level, - regex.error_level ) - if ( max_error_level >= - galaxy.tools.StdioErrorLevel.FATAL ): - break - - if ( regex.stderr_match ): - regex_match = re.search( regex.match, stderr, - re.IGNORECASE ) - if ( regex_match ): - rexmsg = self.regex_err_msg( regex_match, regex) - log.info( "Job %s: %s" - % ( job.get_id_tag(), rexmsg ) ) - stderr = rexmsg + "\n" + stderr - max_error_level = max( max_error_level, - regex.error_level ) - if ( max_error_level >= - galaxy.tools.StdioErrorLevel.FATAL ): - break - - # If we encountered a fatal error, then we'll need to set the - # job state accordingly. Otherwise the job is ok: - if max_error_level >= galaxy.tools.StdioErrorLevel.FATAL: - success = False - else: - success = True - - # When there are no regular expressions and no exit codes to check, - # default to the previous behavior: when there's anything on stderr - # the job has an error, and the job is ok otherwise. - else: - # TODO: Add in the tool and job id: - # log.debug( "Tool did not define exit code or stdio handling; " - # + "checking stderr for success" ) - if stderr: - success = False - else: - success = True - - # On any exception, return True. - except: - tb = traceback.format_exc() - log.warning( "Tool check encountered unexpected exception; " - + "assuming tool was successful: " + tb ) - success = True - - # Store the modified stdout and stderr in the job: - if None != job: - job.stdout = stdout - job.stderr = stderr - - return success - - def regex_err_msg( self, match, regex ): - """ - Return a message about the match on tool output using the given - ToolStdioRegex regex object. The regex_match is a MatchObject - that will contain the string matched on. - """ - # Get the description for the error level: - err_msg = galaxy.tools.StdioErrorLevel.desc( regex.error_level ) + ": " - # If there's a description for the regular expression, then use it. - # Otherwise, we'll take the first 256 characters of the match. - if None != regex.desc: - err_msg += regex.desc - else: - mstart = match.start() - mend = match.end() - err_msg += "Matched on " - # TODO: Move the constant 256 somewhere else besides here. - if mend - mstart > 256: - err_msg += match.string[ mstart : mstart+256 ] + "..." - else: - err_msg += match.string[ mstart: mend ] - return err_msg + return check_output( self.tool, stdout, stderr, tool_exit_code, job ) def cleanup( self ): # remove temporary files diff -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e -r e4d476ccf7832df0b1f65048d3784b010f84e59a lib/galaxy/jobs/error_level.py --- /dev/null +++ b/lib/galaxy/jobs/error_level.py @@ -0,0 +1,25 @@ + + +# These determine stdio-based error levels from matching on regular expressions +# and exit codes. They are meant to be used comparatively, such as showing +# that warning < fatal. This is really meant to just be an enum. +class StdioErrorLevel( object ): + NO_ERROR = 0 + LOG = 1 + WARNING = 2 + FATAL = 3 + MAX = 3 + descs = { + NO_ERROR: 'No error', + LOG: 'Log', + WARNING: 'Warning', + FATAL: 'Fatal error', + } + + @staticmethod + def desc( error_level ): + err_msg = "Unknown error" + if ( error_level > 0 and + error_level <= StdioErrorLevel.MAX ): + err_msg = StdioErrorLevel.descs[ error_level ] + return err_msg diff -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e -r e4d476ccf7832df0b1f65048d3784b010f84e59a lib/galaxy/jobs/output_checker.py --- /dev/null +++ b/lib/galaxy/jobs/output_checker.py @@ -0,0 +1,164 @@ +import re +from .error_level import StdioErrorLevel +import traceback + +from logging import getLogger +log = getLogger( __name__ ) + + +def check_output( tool, stdout, stderr, tool_exit_code, job ): + """ + Check the output of a tool - given the stdout, stderr, and the tool's + exit code, return True if the tool exited succesfully and False + otherwise. No exceptions should be thrown. If this code encounters + an exception, it returns True so that the workflow can continue; + otherwise, a bug in this code could halt workflow progress. + + Note that, if the tool did not define any exit code handling or + any stdio/stderr handling, then it reverts back to previous behavior: + if stderr contains anything, then False is returned. + + Note that the job id is just for messages. + """ + # By default, the tool succeeded. This covers the case where the code + # has a bug but the tool was ok, and it lets a workflow continue. + success = True + + try: + # Check exit codes and match regular expressions against stdout and + # stderr if this tool was configured to do so. + # If there is a regular expression for scanning stdout/stderr, + # then we assume that the tool writer overwrote the default + # behavior of just setting an error if there is *anything* on + # stderr. + if ( len( tool.stdio_regexes ) > 0 or + len( tool.stdio_exit_codes ) > 0 ): + # Check the exit code ranges in the order in which + # they were specified. Each exit_code is a StdioExitCode + # that includes an applicable range. If the exit code was in + # that range, then apply the error level and add a message. + # If we've reached a fatal error rule, then stop. + max_error_level = StdioErrorLevel.NO_ERROR + if tool_exit_code != None: + for stdio_exit_code in tool.stdio_exit_codes: + if ( tool_exit_code >= stdio_exit_code.range_start and + tool_exit_code <= stdio_exit_code.range_end ): + # Tack on a generic description of the code + # plus a specific code description. For example, + # this might prepend "Job 42: Warning (Out of Memory)\n". + code_desc = stdio_exit_code.desc + if ( None == code_desc ): + code_desc = "" + tool_msg = ( "%s: Exit code %d (%s)" % ( + StdioErrorLevel.desc( stdio_exit_code.error_level ), + tool_exit_code, + code_desc ) ) + log.info( "Job %s: %s" % (job.get_id_tag(), tool_msg) ) + stderr = tool_msg + "\n" + stderr + max_error_level = max( max_error_level, + stdio_exit_code.error_level ) + if ( max_error_level >= + StdioErrorLevel.FATAL ): + break + + if max_error_level < StdioErrorLevel.FATAL: + # We'll examine every regex. Each regex specifies whether + # it is to be run on stdout, stderr, or both. (It is + # possible for neither stdout nor stderr to be scanned, + # but those regexes won't be used.) We record the highest + # error level, which are currently "warning" and "fatal". + # If fatal, then we set the job's state to ERROR. + # If warning, then we still set the job's state to OK + # but include a message. We'll do this if we haven't seen + # a fatal error yet + for regex in tool.stdio_regexes: + # If ( this regex should be matched against stdout ) + # - Run the regex's match pattern against stdout + # - If it matched, then determine the error level. + # o If it was fatal, then we're done - break. + # Repeat the stdout stuff for stderr. + # TODO: Collapse this into a single function. + if ( regex.stdout_match ): + regex_match = re.search( regex.match, stdout, + re.IGNORECASE ) + if ( regex_match ): + rexmsg = __regex_err_msg( regex_match, regex) + log.info( "Job %s: %s" + % ( job.get_id_tag(), rexmsg ) ) + stdout = rexmsg + "\n" + stdout + max_error_level = max( max_error_level, + regex.error_level ) + if ( max_error_level >= + StdioErrorLevel.FATAL ): + break + + if ( regex.stderr_match ): + regex_match = re.search( regex.match, stderr, + re.IGNORECASE ) + if ( regex_match ): + rexmsg = __regex_err_msg( regex_match, regex) + log.info( "Job %s: %s" + % ( job.get_id_tag(), rexmsg ) ) + stderr = rexmsg + "\n" + stderr + max_error_level = max( max_error_level, + regex.error_level ) + if ( max_error_level >= + StdioErrorLevel.FATAL ): + break + + # If we encountered a fatal error, then we'll need to set the + # job state accordingly. Otherwise the job is ok: + if max_error_level >= StdioErrorLevel.FATAL: + success = False + else: + success = True + + # When there are no regular expressions and no exit codes to check, + # default to the previous behavior: when there's anything on stderr + # the job has an error, and the job is ok otherwise. + else: + # TODO: Add in the tool and job id: + # log.debug( "Tool did not define exit code or stdio handling; " + # + "checking stderr for success" ) + if stderr: + success = False + else: + success = True + + # On any exception, return True. + except: + tb = traceback.format_exc() + log.warning( "Tool check encountered unexpected exception; " + + "assuming tool was successful: " + tb ) + success = True + + # Store the modified stdout and stderr in the job: + if None != job: + job.stdout = stdout + job.stderr = stderr + + return success + + +def __regex_err_msg( match, regex ): + """ + Return a message about the match on tool output using the given + ToolStdioRegex regex object. The regex_match is a MatchObject + that will contain the string matched on. + """ + # Get the description for the error level: + err_msg = StdioErrorLevel.desc( regex.error_level ) + ": " + # If there's a description for the regular expression, then use it. + # Otherwise, we'll take the first 256 characters of the match. + if None != regex.desc: + err_msg += regex.desc + else: + mstart = match.start() + mend = match.end() + err_msg += "Matched on " + # TODO: Move the constant 256 somewhere else besides here. + if mend - mstart > 256: + err_msg += match.string[ mstart : mstart + 256 ] + "..." + else: + err_msg += match.string[ mstart: mend ] + return err_msg diff -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e -r e4d476ccf7832df0b1f65048d3784b010f84e59a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -34,6 +34,7 @@ from sqlalchemy import and_ from galaxy import jobs, model +from galaxy.jobs.error_level import StdioErrorLevel from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy.jobs import ParallelismInfo from galaxy.tools.actions import DefaultToolAction @@ -64,33 +65,11 @@ from tool_shed.util import shed_util_common from .loader import load_tool, template_macro_params + log = logging.getLogger( __name__ ) WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile( '''\$\{.+?\}''' ) -# These determine stdio-based error levels from matching on regular expressions -# and exit codes. They are meant to be used comparatively, such as showing -# that warning < fatal. This is really meant to just be an enum. -class StdioErrorLevel( object ): - NO_ERROR = 0 - LOG = 1 - WARNING = 2 - FATAL = 3 - MAX = 3 - descs = { - NO_ERROR : 'No error', - LOG: 'Log', - WARNING : 'Warning', - FATAL : 'Fatal error' - } - @staticmethod - def desc( error_level ): - err_msg = "Unknown error" - if ( error_level > 0 and - error_level <= StdioErrorLevel.MAX ): - err_msg = StdioErrorLevel.descs[ error_level ] - return err_msg - class ToolNotFoundException( Exception ): pass diff -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e -r e4d476ccf7832df0b1f65048d3784b010f84e59a test/unit/test_job_output_checker.py --- /dev/null +++ b/test/unit/test_job_output_checker.py @@ -0,0 +1,62 @@ +from unittest import TestCase +from galaxy.util.bunch import Bunch +from galaxy.jobs.output_checker import check_output +from galaxy.jobs.error_level import StdioErrorLevel + + +class OutputCheckerTestCase( TestCase ): + + def setUp( self ): + self.tool = Bunch( + stdio_regexes=[], + stdio_exit_codes=[], + ) + self.job = Bunch( + stdout=None, + stderr=None, + get_id_tag=lambda: "test_id", + ) + self.stdout = '' + self.stderr = '' + self.tool_exit_code = None + + def test_default_no_stderr_success( self ): + self.__assertSuccessful() + + def test_default_stderr_failure( self ): + self.stderr = 'foo' + self.__assertNotSuccessful() + + def test_exit_code_error( self ): + mock_exit_code = Bunch( range_start=1, range_end=1, error_level=StdioErrorLevel.FATAL, desc=None ) + self.tool.stdio_exit_codes.append( mock_exit_code ) + self.tool_exit_code = 1 + self.__assertNotSuccessful() + + def test_exit_code_success( self ): + mock_exit_code = Bunch( range_start=1, range_end=1, error_level=StdioErrorLevel.FATAL, desc=None ) + self.tool.stdio_exit_codes.append( mock_exit_code ) + self.tool_exit_code = 0 + self.__assertSuccessful() + + def test_problematic_strings( self ): + problematic_str = '\x80abc' + regex_rule = Bunch( match=r'.abc', stdout_match=False, stderr_match=True, error_level=StdioErrorLevel.FATAL, desc=None ) + self.tool.stdio_regexes = [ regex_rule ] + self.stderr = problematic_str + self.__assertNotSuccessful() + + problematic_str = '\x80abc' + regex_rule = Bunch( match=r'.abcd', stdout_match=False, stderr_match=True, error_level=StdioErrorLevel.FATAL, desc=None ) + self.tool.stdio_regexes = [ regex_rule ] + self.stderr = problematic_str + self.__assertSuccessful() + + def __assertSuccessful( self ): + self.assertTrue( self.__check_output() ) + + def __assertNotSuccessful( self ): + self.assertFalse( self.__check_output() ) + + def __check_output( self ): + return check_output( self.tool, self.stdout, self.stderr, self.tool_exit_code, self.job ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Remove tipsy because it has been replaced by bootstrap tooltips.
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d966d09f77d9/ Changeset: d966d09f77d9 User: jgoecks Date: 2013-10-16 19:03:47 Summary: Remove tipsy because it has been replaced by bootstrap tooltips. Affected #: 4 files diff -r 2d78abc47928918a14f56711657696724cd9af72 -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e static/scripts/galaxy.base.js --- a/static/scripts/galaxy.base.js +++ b/static/scripts/galaxy.base.js @@ -666,11 +666,6 @@ }); // Tooltips - // if ( $.fn.tipsy ) { - // // FIXME: tipsy gravity cannot be updated, so need classes that specify N/S gravity and - // // initialize each separately. - // $(".tooltip").tipsy( { gravity: 's' } ); - // } if ( $.fn.tooltip ) { // Put tooltips below items in panel header so that they do not overlap masthead. $(".unified-panel-header [title]").tooltip( { placement: 'bottom' } ); diff -r 2d78abc47928918a14f56711657696724cd9af72 -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e static/scripts/libs/jquery/jquery.tipsy.js --- a/static/scripts/libs/jquery/jquery.tipsy.js +++ /dev/null @@ -1,172 +0,0 @@ -/* NOTE: MODIFIED FROM ORIGINAL! */ - -(function($) { - function fixTitle($ele) { - if ($ele.attr('title') || typeof($ele.attr('original-title')) != 'string') { - $ele.attr('original-title', $ele.attr('title') || '').removeAttr('title'); - } - } - - $.fn.tipsy = function(options) { - - options = $.extend({}, $.fn.tipsy.defaults, options); - - return this.each(function() { - - fixTitle($(this)); - var opts = $.fn.tipsy.elementOptions(this, options); - var timeout = null; - - $(this).hover(function() { - var self = this; - timeout = setTimeout(function() { - $.data(self, 'cancel.tipsy', true); - - var tip = $.data(self, 'active.tipsy'); - if (!tip) { - tip = $('<div class="tipsy"><div class="tipsy-inner"/></div>'); - tip.css({position: 'absolute', zIndex: 100000}); - $.data(self, 'active.tipsy', tip); - } - - fixTitle($(self)); - - var title; - if (typeof opts.title == 'string') { - title = $(self).attr(opts.title == 'title' ? 'original-title' : opts.title); - } else if (typeof opts.title == 'function') { - title = opts.title.call(self); - } - - tip.find('.tipsy-inner')[opts.html ? 'html' : 'text'](title || opts.fallback); - - - var pos = $.extend({}, $(self).offset(), {width: self.offsetWidth, height: self.offsetHeight}); - tip.get(0).className = 'tipsy'; // reset classname in case of dynamic gravity - tip.remove().css({top: 0, left: 0, visibility: 'hidden', display: 'block'}).appendTo(document.body); - - tip.css( { width: tip.width() + 1, height: tip.height() } ); - - var actualWidth = tip[0].offsetWidth, actualHeight = tip[0].offsetHeight; - var gravity = (typeof opts.gravity == 'function') ? opts.gravity.call(self) : opts.gravity; - - var top, left; - switch (gravity.charAt(0)) { - case 'n': - top = pos.top + pos.height; - left = pos.left + pos.width / 2 - actualWidth / 2; - tip.addClass('tipsy-north'); - break; - case 's': - top = pos.top - actualHeight; - left = pos.left + pos.width / 2 - actualWidth / 2; - tip.addClass('tipsy-south'); - break; - case 'e': - top = pos.top + pos.height / 2 - actualHeight / 2; - left = pos.left - actualWidth; - tip.addClass('tipsy-east'); - break; - case 'w': - top = pos.top + pos.height / 2 - actualHeight / 2; - left = pos.left + pos.width; - tip.addClass('tipsy-west'); - break; - } - // Shift if off screen - var w = $(window); - - // If off the top of the screen, flip - if ( top < w.scrollTop() && gravity.charAt( 0 ) == 's' ) { - top = pos.top + pos.height; - gravity = 'north'; - tip.removeClass('tipsy-south').addClass('tipsy-north'); - } - - // If off bottom, just shift for now - top = Math.min( top, w.scrollTop() + w.height() - tip.outerHeight() ); - - - // Shift left or right - var left_shift = 0; - if ( left < w.scrollLeft() ) { - left_shift = left - w.scrollLeft(); - } - var t = w.scrollLeft() + w.width() - tip.outerWidth(); - if ( left > t ) { - left_shift = left - t; - } - - left -= left_shift; - - tip.css( { left: left, top: top } ); - - // Shift background to center over element (not implemented for east/west) - switch (gravity.charAt(0)) { - case 'n': - tip.css( 'background-position', - ( 250 - tip.outerWidth() / 2 ) + left_shift + "px top" ); - break; - case 's': - tip.css( 'background-position', - ( 250 - tip.outerWidth() / 2 ) + left_shift + "px bottom" ); - break; - case 'e': - break; - case 'w': - break; - } - - if (opts.fade) { - tip.stop().css({opacity: 0, display: 'block', visibility: 'visible'}).animate({opacity: opts.opacity}); - } else { - tip.css({visibility: 'visible', opacity: opts.opacity}); - } - }, opts.delayIn); - - }, function() { - $.data(this, 'cancel.tipsy', false); - var self = this; - clearTimeout(timeout); - setTimeout(function() { - if ($.data(this, 'cancel.tipsy')) { return; } - var tip = $.data(self, 'active.tipsy'); - if (opts.fade) { - tip.stop().fadeOut(function() { $(this).remove(); }); - } else if (tip) { - tip.remove(); - } - }, opts.delayOut); - - }); - - }); - - }; - - // Overwrite this method to provide options on a per-element basis. - // For example, you could store the gravity in a 'tipsy-gravity' attribute: - // return $.extend({}, options, {gravity: $(ele).attr('tipsy-gravity') || 'n' }); - // (remember - do not modify 'options' in place!) - $.fn.tipsy.elementOptions = function(ele, options) { - return $.metadata ? $.extend({}, options, $(ele).metadata()) : options; - }; - - $.fn.tipsy.defaults = { - delayIn: 0, - delayOut: 100, - fade: false, - fallback: '', - gravity: 'n', - html: false, - opacity: 0.8, - title: 'title' - }; - - $.fn.tipsy.autoNS = function() { - return $(this).offset().top > ($(document).scrollTop() + $(window).height() / 2) ? 's' : 'n'; - }; - - $.fn.tipsy.autoWE = function() { - return $(this).offset().left > ($(document).scrollLeft() + $(window).width() / 2) ? 'e' : 'w'; - }; - -})(jQuery); diff -r 2d78abc47928918a14f56711657696724cd9af72 -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e static/scripts/packed/libs/jquery/jquery.tipsy.js --- a/static/scripts/packed/libs/jquery/jquery.tipsy.js +++ /dev/null @@ -1,1 +0,0 @@ -(function(b){function a(c){if(c.attr("title")||typeof(c.attr("original-title"))!="string"){c.attr("original-title",c.attr("title")||"").removeAttr("title")}}b.fn.tipsy=function(c){c=b.extend({},b.fn.tipsy.defaults,c);return this.each(function(){a(b(this));var d=b.fn.tipsy.elementOptions(this,c);var e=null;b(this).hover(function(){var f=this;e=setTimeout(function(){b.data(f,"cancel.tipsy",true);var o=b.data(f,"active.tipsy");if(!o){o=b('<div class="tipsy"><div class="tipsy-inner"/></div>');o.css({position:"absolute",zIndex:100000});b.data(f,"active.tipsy",o)}a(b(f));var m;if(typeof d.title=="string"){m=b(f).attr(d.title=="title"?"original-title":d.title)}else{if(typeof d.title=="function"){m=d.title.call(f)}}o.find(".tipsy-inner")[d.html?"html":"text"](m||d.fallback);var k=b.extend({},b(f).offset(),{width:f.offsetWidth,height:f.offsetHeight});o.get(0).className="tipsy";o.remove().css({top:0,left:0,visibility:"hidden",display:"block"}).appendTo(document.body);o.css({width:o.width()+1,height:o.height()});var h=o[0].offsetWidth,j=o[0].offsetHeight;var q=(typeof d.gravity=="function")?d.gravity.call(f):d.gravity;var l,i;switch(q.charAt(0)){case"n":l=k.top+k.height;i=k.left+k.width/2-h/2;o.addClass("tipsy-north");break;case"s":l=k.top-j;i=k.left+k.width/2-h/2;o.addClass("tipsy-south");break;case"e":l=k.top+k.height/2-j/2;i=k.left-h;o.addClass("tipsy-east");break;case"w":l=k.top+k.height/2-j/2;i=k.left+k.width;o.addClass("tipsy-west");break}var n=b(window);if(l<n.scrollTop()&&q.charAt(0)=="s"){l=k.top+k.height;q="north";o.removeClass("tipsy-south").addClass("tipsy-north")}l=Math.min(l,n.scrollTop()+n.height()-o.outerHeight());var g=0;if(i<n.scrollLeft()){g=i-n.scrollLeft()}var p=n.scrollLeft()+n.width()-o.outerWidth();if(i>p){g=i-p}i-=g;o.css({left:i,top:l});switch(q.charAt(0)){case"n":o.css("background-position",-(250-o.outerWidth()/2)+g+"px top");break;case"s":o.css("background-position",-(250-o.outerWidth()/2)+g+"px bottom");break;case"e":break;case"w":break}if(d.fade){o.stop().css({opacity:0,display:"block",visibility:"visible"}).animate({opacity:d.opacity})}else{o.css({visibility:"visible",opacity:d.opacity})}},d.delayIn)},function(){b.data(this,"cancel.tipsy",false);var f=this;clearTimeout(e);setTimeout(function(){if(b.data(this,"cancel.tipsy")){return}var g=b.data(f,"active.tipsy");if(d.fade){g.stop().fadeOut(function(){b(this).remove()})}else{if(g){g.remove()}}},d.delayOut)})})};b.fn.tipsy.elementOptions=function(d,c){return b.metadata?b.extend({},c,b(d).metadata()):c};b.fn.tipsy.defaults={delayIn:0,delayOut:100,fade:false,fallback:"",gravity:"n",html:false,opacity:0.8,title:"title"};b.fn.tipsy.autoNS=function(){return b(this).offset().top>(b(document).scrollTop()+b(window).height()/2)?"s":"n"};b.fn.tipsy.autoWE=function(){return b(this).offset().left>(b(document).scrollLeft()+b(window).width()/2)?"e":"w"}})(jQuery); \ No newline at end of file diff -r 2d78abc47928918a14f56711657696724cd9af72 -r d966d09f77d9a3fed4401eb1e5e64d17a57d684e static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -292,7 +292,7 @@ title: "Remove", css_class: "remove-icon", on_click_fn: function(drawable) { - // Tipsy for remove icon must be deleted when drawable is deleted. + // Tooltip for remove icon must be deleted when drawable is deleted. $(".tooltip").remove(); drawable.remove(); } @@ -642,7 +642,7 @@ title: "Filters", css_class: "filters-icon", on_click_fn: function(group) { - // TODO: update tipsy text. + // TODO: update Tooltip text. if (group.filters_manager.visible()) { // Hiding filters. group.filters_manager.clear_filters(); @@ -2409,7 +2409,7 @@ title: "Filters", css_class: "filters-icon", on_click_fn: function(drawable) { - // TODO: update tipsy text. + // TODO: update Tooltip text. if (drawable.filters_manager.visible()) { drawable.filters_manager.clear_filters(); } @@ -2425,7 +2425,7 @@ title: "Tool", css_class: "hammer", on_click_fn: function(track) { - // TODO: update tipsy text. + // TODO: update Tooltip text. track.tool.toggle(); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: martenson: honeypot trap for bot registering accounts
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2d78abc47928/ Changeset: 2d78abc47928 User: martenson Date: 2013-10-16 17:40:02 Summary: honeypot trap for bot registering accounts Affected #: 2 files diff -r d3d4210d0abee42c4c7cb8d66b8a325f340899c1 -r 2d78abc47928918a14f56711657696724cd9af72 lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -594,6 +594,12 @@ @web.expose def create( self, trans, cntrller='user', redirect_url='', refresh_frames=[], **kwd ): params = util.Params( kwd ) + + # If the honeypot field is not empty we are dealing with a bot. + honeypot_field = params.get( 'bear_field', '' ) + if honeypot_field != '': + return trans.show_error_message( "You are considered a bot. If you are not one please try registering again and follow the form's legend. <a target=\"_top\" href=\"%s\">Go to the home page</a>." ) % url_for( '/' ) + message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) use_panels = util.string_as_bool( kwd.get( 'use_panels', True ) ) diff -r d3d4210d0abee42c4c7cb8d66b8a325f340899c1 -r 2d78abc47928918a14f56711657696724cd9af72 templates/user/register.mako --- a/templates/user/register.mako +++ b/templates/user/register.mako @@ -162,7 +162,7 @@ %endif <div id="for_bears"> If you see this, please leave following field blank. - <input type="text" name="please leave this field blank" size="1" value=""/> + <input type="text" name="bear_field" size="1" value=""/></div><div class="form-row"><input type="submit" id="send" name="create_user_button" value="Submit"/> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Modify bowtie error handler
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d3d4210d0abe/ Changeset: d3d4210d0abe User: guerler Date: 2013-10-16 17:03:51 Summary: Modify bowtie error handler Affected #: 3 files diff -r 940a310179c4ad9cfe7dabae2ac71402608a636a -r d3d4210d0abee42c4c7cb8d66b8a325f340899c1 static/style/blue/base.css --- a/static/style/blue/base.css +++ b/static/style/blue/base.css @@ -1129,7 +1129,7 @@ .upload-box .table th{text-align:center;white-space:nowrap} .upload-box .table td{margin:0px;paddign:0px} .upload-box .title{width:130px;word-wrap:break-word;font-size:11px} -.upload-box .text{position:absolute;display:none}.upload-box .text .text-content{font-size:11px;width:100%;height:50px;resize:none;background:inherit} +.upload-box .text{position:absolute;display:none}.upload-box .text .text-content{font-size:11px;width:100%;height:50px;resize:none;background:inherit;color:#000} .upload-box .text .text-info{font-size:11px;color:#999} .upload-box .extension{width:100px;font-size:11px} .upload-box .genome{width:150px;font-size:11px} diff -r 940a310179c4ad9cfe7dabae2ac71402608a636a -r d3d4210d0abee42c4c7cb8d66b8a325f340899c1 static/style/src/less/upload.less --- a/static/style/src/less/upload.less +++ b/static/style/src/less/upload.less @@ -45,10 +45,11 @@ .text-content { font-size : @font-size-small; - width : 100%; - height : 50px; - resize : none; - background : inherit; + width: 100%; + height: 50px; + resize: none; + background: inherit; + color: @black; } .text-info { diff -r 940a310179c4ad9cfe7dabae2ac71402608a636a -r d3d4210d0abee42c4c7cb8d66b8a325f340899c1 tools/sr_mapping/bowtie2_wrapper.xml --- a/tools/sr_mapping/bowtie2_wrapper.xml +++ b/tools/sr_mapping/bowtie2_wrapper.xml @@ -109,7 +109,7 @@ <stdio> - <regex match="Exception" source="stderr" level="fatal" description="Tool exception"/> + <exit_code range="1:" level="fatal" description="Tool exception" /></stdio><inputs> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0