January 2015 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: guerler: ToolForm: Add additional trigger for refresh event
by commits-noreply＠bitbucket.org 13 Jan '15

13 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ba17da4b2356/ Changeset: ba17da4b2356 User: guerler Date: 2015-01-14 05:52:19+00:00 Summary: ToolForm: Add additional trigger for refresh event Affected #: 3 files diff -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc -r ba17da4b23564fdff554c6f0b5d0618326147b97 client/galaxy/scripts/mvc/tools/tools-form-base.js --- a/client/galaxy/scripts/mvc/tools/tools-form-base.js +++ b/client/galaxy/scripts/mvc/tools/tools-form-base.js @@ -131,6 +131,9 @@ this.element_list[i].reset(); } }); + + // refresh + this.trigger('refresh'); }, /** Renders the UI elements required for the form diff -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc -r ba17da4b23564fdff554c6f0b5d0618326147b97 static/scripts/mvc/tools/tools-form-base.js --- a/static/scripts/mvc/tools/tools-form-base.js +++ b/static/scripts/mvc/tools/tools-form-base.js @@ -131,6 +131,9 @@ this.element_list[i].reset(); } }); + + // refresh + this.trigger('refresh'); }, /** Renders the UI elements required for the form diff -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc -r ba17da4b23564fdff554c6f0b5d0618326147b97 static/scripts/packed/mvc/tools/tools-form-base.js --- a/static/scripts/packed/mvc/tools/tools-form-base.js +++ b/static/scripts/packed/mvc/tools/tools-form-base.js @@ -1,1 +1,1 @@ -define(["utils/utils","utils/deferred","mvc/ui/ui-portlet","mvc/ui/ui-misc","mvc/citation/citation-model","mvc/citation/citation-view","mvc/tools","mvc/tools/tools-template","mvc/tools/tools-content","mvc/tools/tools-section","mvc/tools/tools-tree"],function(g,h,f,k,i,a,d,c,e,j,b){return Backbone.View.extend({initialize:function(l){console.debug(l);var m=parent.Galaxy;if(m&&m.modal){this.modal=m.modal}else{this.modal=new k.Modal.View()}if(m&&m.currUser){this.is_admin=m.currUser.get("is_admin")}else{this.is_admin=false}this.options=l;this.container=this.options.container||"body";this.deferred=new h();this.setElement("<div/>");$(this.container).append(this.$el);this._buildForm()},reciept:function(l){$(this.container).empty();$(this.container).append(l)},highlight:function(m,n,l){var o=this.element_list[m];if(o){o.error(n||"Please verify this parameter.");if(!l){$(this.container).animate({scrollTop:o.$el.offset().top-20},500)}}},_buildForm:function(){var l=this;this.off("refresh");this.off("reset");this.field_list={};this.input_list={};this.element_list={};this.tree=new b(this);this.content=new e(this);var n=this.options;this._renderForm(n);this.tree.finalize();if(n.errors){var o=this.tree.matchResponse(n.errors);for(var m in o){this.highlight(m,o[m],true)}}this.on("refresh",function(){l.deferred.reset();l.deferred.execute(function(){l._updateModel()})});this.on("reset",function(){for(var p in this.element_list){this.element_list[p].reset()}})},_renderForm:function(u){var t=this;this.message=new k.Message();var q=new k.ButtonIcon({icon:"fa-info-circle",title:(!t.workflow&&"Requirements")||null,tooltip:"Display tool requirements",onclick:function(){if(!this.visible){this.visible=true;t.message.update({persistent:true,message:c.requirements(u),status:"info"})}else{this.visible=false;t.message.update({message:""})}}});if(!u.requirements||u.requirements.length==0){q.$el.hide()}var m=new k.ButtonMenu({icon:"fa-cubes",title:(!t.workflow&&"Versions")||null,tooltip:"Select another tool version"});if(u.versions&&u.versions.length>1){for(var o in u.versions){var r=u.versions[o];if(r!=u.version){m.addMenu({title:"Switch to "+r,version:r,icon:"fa-cube",onclick:function(){u.id=u.id.replace(u.version,this.version);u.version=this.version;t.deferred.reset();t.deferred.execute(function(){t._buildModel()})}})}}}else{m.$el.hide()}var p=new k.ButtonMenu({icon:"fa-caret-down",title:(!t.workflow&&"Options")||null,tooltip:"View available options"});if(u.biostar_url){p.addMenu({icon:"fa-question-circle",title:"Question?",tooltip:"Ask a question about this tool (Biostar)",onclick:function(){window.open(u.biostar_url+"/p/new/post/")}});p.addMenu({icon:"fa-search",title:"Search",tooltip:"Search help for this tool (Biostar)",onclick:function(){window.open(u.biostar_url+"/t/"+u.id+"/")}})}p.addMenu({icon:"fa-share",title:"Share",tooltip:"Share this tool",onclick:function(){prompt("Copy to clipboard: Ctrl+C, Enter",window.location.origin+galaxy_config.root+"root?tool_id="+u.id)}});if(this.is_admin){p.addMenu({icon:"fa-download",title:"Download",tooltip:"Download this tool",onclick:function(){window.location.href=galaxy_config.root+"api/tools/"+u.id+"/download"}})}this.section=new j.View(t,{inputs:u.inputs,cls:"ui-table-plain"});if(this.incompatible){this.$el.hide();$("#tool-form-classic").show();return}this.portlet=new f.View({icon:"fa-wrench",title:"<b>"+u.name+"</b> "+u.description+" (Galaxy Tool Version "+u.version+")",cls:"ui-portlet-slim",operations:{requirements:q,menu:p,versions:m},buttons:this.buttons});if(this.options.workflow){this.portlet.$content.css("padding","0px")}this.portlet.append(this.message.$el,true);this.portlet.append(this.section.$el);this.$el.empty();this.$el.append(this.portlet.$el);if(u.help!=""){this.$el.append(c.help(u.help))}if(u.citations){var s=$("<div/>");var l=new i.ToolCitationCollection();l.tool_id=u.id;var n=new a.CitationListView({el:s,collection:l});n.render();l.fetch();this.$el.append(s)}if(u.message){this.message.update({persistent:true,status:"warning",message:u.message})}}})}); \ No newline at end of file +define(["utils/utils","utils/deferred","mvc/ui/ui-portlet","mvc/ui/ui-misc","mvc/citation/citation-model","mvc/citation/citation-view","mvc/tools","mvc/tools/tools-template","mvc/tools/tools-content","mvc/tools/tools-section","mvc/tools/tools-tree"],function(g,h,f,k,i,a,d,c,e,j,b){return Backbone.View.extend({initialize:function(l){console.debug(l);var m=parent.Galaxy;if(m&&m.modal){this.modal=m.modal}else{this.modal=new k.Modal.View()}if(m&&m.currUser){this.is_admin=m.currUser.get("is_admin")}else{this.is_admin=false}this.options=l;this.container=this.options.container||"body";this.deferred=new h();this.setElement("<div/>");$(this.container).append(this.$el);this._buildForm()},reciept:function(l){$(this.container).empty();$(this.container).append(l)},highlight:function(m,n,l){var o=this.element_list[m];if(o){o.error(n||"Please verify this parameter.");if(!l){$(this.container).animate({scrollTop:o.$el.offset().top-20},500)}}},_buildForm:function(){var l=this;this.off("refresh");this.off("reset");this.field_list={};this.input_list={};this.element_list={};this.tree=new b(this);this.content=new e(this);var n=this.options;this._renderForm(n);this.tree.finalize();if(n.errors){var o=this.tree.matchResponse(n.errors);for(var m in o){this.highlight(m,o[m],true)}}this.on("refresh",function(){l.deferred.reset();l.deferred.execute(function(){l._updateModel()})});this.on("reset",function(){for(var p in this.element_list){this.element_list[p].reset()}});this.trigger("refresh")},_renderForm:function(u){var t=this;this.message=new k.Message();var q=new k.ButtonIcon({icon:"fa-info-circle",title:(!t.workflow&&"Requirements")||null,tooltip:"Display tool requirements",onclick:function(){if(!this.visible){this.visible=true;t.message.update({persistent:true,message:c.requirements(u),status:"info"})}else{this.visible=false;t.message.update({message:""})}}});if(!u.requirements||u.requirements.length==0){q.$el.hide()}var m=new k.ButtonMenu({icon:"fa-cubes",title:(!t.workflow&&"Versions")||null,tooltip:"Select another tool version"});if(u.versions&&u.versions.length>1){for(var o in u.versions){var r=u.versions[o];if(r!=u.version){m.addMenu({title:"Switch to "+r,version:r,icon:"fa-cube",onclick:function(){u.id=u.id.replace(u.version,this.version);u.version=this.version;t.deferred.reset();t.deferred.execute(function(){t._buildModel()})}})}}}else{m.$el.hide()}var p=new k.ButtonMenu({icon:"fa-caret-down",title:(!t.workflow&&"Options")||null,tooltip:"View available options"});if(u.biostar_url){p.addMenu({icon:"fa-question-circle",title:"Question?",tooltip:"Ask a question about this tool (Biostar)",onclick:function(){window.open(u.biostar_url+"/p/new/post/")}});p.addMenu({icon:"fa-search",title:"Search",tooltip:"Search help for this tool (Biostar)",onclick:function(){window.open(u.biostar_url+"/t/"+u.id+"/")}})}p.addMenu({icon:"fa-share",title:"Share",tooltip:"Share this tool",onclick:function(){prompt("Copy to clipboard: Ctrl+C, Enter",window.location.origin+galaxy_config.root+"root?tool_id="+u.id)}});if(this.is_admin){p.addMenu({icon:"fa-download",title:"Download",tooltip:"Download this tool",onclick:function(){window.location.href=galaxy_config.root+"api/tools/"+u.id+"/download"}})}this.section=new j.View(t,{inputs:u.inputs,cls:"ui-table-plain"});if(this.incompatible){this.$el.hide();$("#tool-form-classic").show();return}this.portlet=new f.View({icon:"fa-wrench",title:"<b>"+u.name+"</b> "+u.description+" (Galaxy Tool Version "+u.version+")",cls:"ui-portlet-slim",operations:{requirements:q,menu:p,versions:m},buttons:this.buttons});if(this.options.workflow){this.portlet.$content.css("padding","0px")}this.portlet.append(this.message.$el,true);this.portlet.append(this.section.$el);this.$el.empty();this.$el.append(this.portlet.$el);if(u.help!=""){this.$el.append(c.help(u.help))}if(u.citations){var s=$("<div/>");var l=new i.ToolCitationCollection();l.tool_id=u.id;var n=new a.CitationListView({el:s,collection:l});n.render();l.fetch();this.$el.append(s)}if(u.message){this.message.update({persistent:true,status:"warning",message:u.message})}}})}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: ToolForm/Workflow: Add payload, use default value to collapse/expand sections, fix styles
by commits-noreply＠bitbucket.org 13 Jan '15

13 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/bf6b51623d29/ Changeset: bf6b51623d29 User: guerler Date: 2015-01-14 03:39:17+00:00 Summary: ToolForm/Workflow: Add payload, use default value to collapse/expand sections, fix styles Affected #: 25 files diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc client/galaxy/scripts/mvc/tools/tools-form-base.js --- a/client/galaxy/scripts/mvc/tools/tools-form-base.js +++ b/client/galaxy/scripts/mvc/tools/tools-form-base.js @@ -111,7 +111,7 @@ this.tree.finalize(); // show errors - if (!this.workflow && options.errors) { + if (options.errors) { var error_messages = this.tree.matchResponse(options.errors); for (var input_id in error_messages) { this.highlight(input_id, error_messages[input_id], true); @@ -145,7 +145,7 @@ // button for version selection var requirements_button = new Ui.ButtonIcon({ icon : 'fa-info-circle', - title : 'Requirements', + title : (!self.workflow && 'Requirements') || null, tooltip : 'Display tool requirements', onclick : function() { if (!this.visible) { @@ -170,7 +170,7 @@ // button for version selection var versions_button = new Ui.ButtonMenu({ icon : 'fa-cubes', - title : 'Versions', + title : (!self.workflow && 'Versions') || null, tooltip : 'Select another tool version' }); if (options.versions && options.versions.length > 1) { @@ -200,7 +200,7 @@ // button menu var menu_button = new Ui.ButtonMenu({ icon : 'fa-caret-down', - title : 'Options', + title : (!self.workflow && 'Options') || null, tooltip : 'View available options' }); diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc client/galaxy/scripts/mvc/tools/tools-form-workflow.js --- a/client/galaxy/scripts/mvc/tools/tools-form-workflow.js +++ b/client/galaxy/scripts/mvc/tools/tools-form-workflow.js @@ -9,6 +9,7 @@ initialize: function(options) { // link with node representation in workflow module this.node = workflow.active_node; + if (!this.node) { console.debug('FAILED - tools-form-workflow:initialize() - Node not found in workflow.'); return; @@ -18,6 +19,13 @@ this.workflow = true; this.options = options; + // traverse through dictionary + Utils.deepeach(options.inputs, function(item) { + if (item.type && (['data', 'data_hidden', 'hidden']).indexOf(item.type) == -1) { + item.optional = true; + } + }); + // load extension var self = this; Utils.get({ @@ -38,6 +46,7 @@ inputs[Utils.uuid()] = { label : 'Edit Step Attributes', type : 'section', + expand : this.node.annotation, inputs : [{ label : 'Annotation / Notes', name : 'annotation', @@ -160,7 +169,10 @@ type : 'boolean', value : 'false', ignore : 'false', - help : 'This action will send an email notifying you when the job is done.' + help : 'This action will send an email notifying you when the job is done.', + payload : { + 'host' : window.location.host + } },{ action : 'DeleteIntermediatesAction', label : 'Delete non-outputs', @@ -173,31 +185,51 @@ // visit input nodes and enrich by name/value pairs from server data var self = this; - function visit (inputs) { - for (var i in inputs) { - var input = inputs[i]; + function visit (head, head_list) { + head_list = head_list || []; + head_list.push(head); + for (var i in head.inputs) { + var input = head.inputs[i]; if (input.action) { + // construct identifier as expected by backend input.name = 'pja__' + output_id + '__' + input.action; if (input.argument) { input.name += '__' + input.argument; } + + // modify names of payload arguments + if (input.payload) { + for (var p_id in input.payload) { + var p = input.payload[p_id]; + input.payload[input.name + '__' + p_id] = p; + delete p; + } + } + + // access/verify existence of value var d = self.post_job_actions[input.action + output_id]; if (d) { + // mark as expanded + for (var j in head_list) { + head_list[j].expand = true; + } + + // update input field value if (input.argument) { input.value = d.action_arguments && d.action_arguments[input.argument] || input.value; } else { input.value = 'true'; } } - } else { - if (input.inputs) { - visit(input.inputs); - } + } + // continue with sub section + if (input.inputs) { + visit(input, head_list.slice(0)); } } } - visit(input_config.inputs); - + visit(input_config); + // return final configuration return input_config; }, @@ -205,6 +237,15 @@ /** Builds a new model through api call and recreates the entire form */ _buildModel: function() { + Galaxy.modal.show({ + title : 'Coming soon...', + body : 'This feature has not been implemented yet.', + buttons : { + 'Close' : function() { + Galaxy.modal.hide(); + } + } + }); }, /** Request a new model for an already created tool form and updates the form inputs diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc client/galaxy/scripts/mvc/tools/tools-input.js --- a/client/galaxy/scripts/mvc/tools/tools-input.js +++ b/client/galaxy/scripts/mvc/tools/tools-input.js @@ -12,6 +12,7 @@ // link field this.field = options.field; + this.defaultvalue = options.defaultvalue; // set element this.setElement(this._template(options)); @@ -25,9 +26,17 @@ // add field element this.$field.prepend(this.field.$el); - // start with enabled optional fields + // decide wether to expand or collapse optional fields this.field.skip = false; - + if (options.optional) { + if ((this.field.validate && !this.field.validate()) || + (this.field.value && !this.field.value()) || + (this.field.value && Number(this.field.value()) == Number(this.defaultvalue)) || + (this.field.value && JSON.stringify(this.field.value()) == JSON.stringify(this.defaultvalue))) { + this.field.skip = true; + } + } + // refresh view this._refresh(); @@ -63,11 +72,12 @@ if (!this.field.skip) { this.$field.fadeIn('fast'); this.$title_optional.html('Disable'); - this.app.trigger('refresh'); } else { this.$field.hide(); this.$title_optional.html('Enable'); + this.field.value(this.defaultvalue); } + this.app.trigger('refresh'); }, /** Main Template diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc client/galaxy/scripts/mvc/tools/tools-section.js --- a/client/galaxy/scripts/mvc/tools/tools-section.js +++ b/client/galaxy/scripts/mvc/tools/tools-section.js @@ -232,9 +232,9 @@ // create input field wrapper var input_element = new InputElement(this.app, { - label : input_def.title, - help : input_def.help, - field : repeat + label : input_def.title, + help : input_def.help, + field : repeat }); // displays as grouped subsection @@ -292,6 +292,11 @@ } }); + // show sub section if requested + if (input_def.expand) { + portlet.$header.trigger('click'); + } + // create table row this.table.add(portlet.$el); @@ -313,10 +318,11 @@ // create input field wrapper var input_element = new InputElement(this.app, { - label : input_def.label, - optional : input_def.optional, - help : input_def.help, - field : field + label : input_def.label, + defaultvalue : input_def.defaultvalue, + optional : input_def.optional, + help : input_def.help, + field : field }); // add to element list diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc client/galaxy/scripts/mvc/tools/tools-template.js --- a/client/galaxy/scripts/mvc/tools/tools-template.js +++ b/client/galaxy/scripts/mvc/tools/tools-template.js @@ -4,7 +4,7 @@ // tool form templates return { help: function(content) { - return '<div class="toolHelp">' + + return '<div class="toolHelp" style="overflow: auto;">' + '<div class="toolHelpBody">' + content + '</div>' + diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc client/galaxy/scripts/mvc/tools/tools-tree.js --- a/client/galaxy/scripts/mvc/tools/tools-tree.js +++ b/client/galaxy/scripts/mvc/tools/tools-tree.js @@ -122,13 +122,21 @@ value = patch[input.type](value); } - // handle default value + // handle simple value if (!field.skip) { - if (field.validate && !field.validate(value)) { + if (field.validate && !field.validate()) { value = null; } - if (input.ignore === undefined || (value && input.ignore != value)) { + if (input.ignore === undefined || (value !== null && input.ignore != value)) { + // add value to submission add (job_input_id, input.id, value); + + // add payload to submission + if (input.payload) { + for (var p_id in input.payload) { + add (p_id, input.id, input.payload[p_id]); + } + } } } } diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc client/galaxy/scripts/utils/utils.js --- a/client/galaxy/scripts/utils/utils.js +++ b/client/galaxy/scripts/utils/utils.js @@ -6,6 +6,18 @@ // dependencies define(["libs/underscore"], function(_) { +/** Traverse through json +*/ +function deepeach(dict, callback) { + for (var i in dict) { + var d = dict[i]; + if (d && typeof(d) == "object") { + callback(d); + deepeach(d, callback); + } + } +} + /** * Sanitize/escape a string * @param{String} content - Content to be sanitized @@ -248,7 +260,8 @@ request: request, sanitize: sanitize, textify: textify, - validate: validate + validate: validate, + deepeach: deepeach }; }); diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -12,8 +12,10 @@ import threading import types import urllib +import copy -from galaxy import eggs +from galaxy import eggs, util + eggs.require( "MarkupSafe" ) # MarkupSafe must load before mako eggs.require( "Mako" ) eggs.require( "elementtree" ) @@ -2150,9 +2152,9 @@ return None # ensures that input dictionary is jsonifiable - def sanitize(dict): + def sanitize(dict, key='value'): # get current value - value = dict['value'] if 'value' in dict else None + value = dict[key] if key in dict else None # jsonify by type if dict['type'] in ['data']: @@ -2167,15 +2169,8 @@ value = jsonify(value) # update and return - dict['value'] = value - return dict + dict[key] = value - # initialize state using default parameters - def initialize_state(trans, inputs, state, context=None): - context = ExpressionContext(state, context) - for input in inputs.itervalues(): - state[input.name] = input.get_initial_value(trans, context) - # check the current state of a value and update it if necessary def check_state(trans, input, default_value, context): value = default_value @@ -2195,6 +2190,7 @@ def populate_state(trans, inputs, state, errors, incoming, prefix="", context=None ): context = ExpressionContext(state, context) for input in inputs.itervalues(): + state[input.name] = input.get_initial_value(trans, context) key = prefix + input.name if input.type == 'repeat': group_state = state[input.name] @@ -2207,7 +2203,6 @@ if rep_index < input.max: new_state = {} new_state['__index__'] = rep_index - initialize_state(trans, input.inputs, new_state, context) group_state.append(new_state) populate_state(trans, input.inputs, new_state, errors, incoming, prefix=rep_name + "|", context=context) rep_index += 1 @@ -2223,7 +2218,6 @@ try: current_case = input.get_current_case(value, trans) group_state = state[input.name] = {} - initialize_state(trans, input.cases[current_case].inputs, group_state, context) populate_state( trans, input.cases[current_case].inputs, group_state, errors, incoming, prefix=group_prefix, context=context) group_state['__current_case__'] = current_case except Exception, e: @@ -2272,13 +2266,17 @@ # identify name input_name = tool_dict.get('name') if input_name: + # backup default value + tool_dict['defaultvalue'] = input.get_initial_value(trans, other_values) + # update input value from tool state if input_name in state_inputs: tool_dict['value'] = state_inputs[input_name] - - # sanitize if value exists - tool_dict = sanitize(tool_dict) - + + # sanitize values + sanitize(tool_dict, 'value') + sanitize(tool_dict, 'defaultvalue') + # backup final input dictionary group_inputs[input_index] = tool_dict @@ -2303,7 +2301,6 @@ # initialize and populate tool state state_inputs = {} state_errors = {} - initialize_state(trans, self.inputs, state_inputs) populate_state(trans, self.inputs, state_inputs, state_errors, params.__dict__) # create basic tool model diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/mvc/tools/tools-form-base.js --- a/static/scripts/mvc/tools/tools-form-base.js +++ b/static/scripts/mvc/tools/tools-form-base.js @@ -111,7 +111,7 @@ this.tree.finalize(); // show errors - if (!this.workflow && options.errors) { + if (options.errors) { var error_messages = this.tree.matchResponse(options.errors); for (var input_id in error_messages) { this.highlight(input_id, error_messages[input_id], true); @@ -145,7 +145,7 @@ // button for version selection var requirements_button = new Ui.ButtonIcon({ icon : 'fa-info-circle', - title : 'Requirements', + title : (!self.workflow && 'Requirements') || null, tooltip : 'Display tool requirements', onclick : function() { if (!this.visible) { @@ -170,7 +170,7 @@ // button for version selection var versions_button = new Ui.ButtonMenu({ icon : 'fa-cubes', - title : 'Versions', + title : (!self.workflow && 'Versions') || null, tooltip : 'Select another tool version' }); if (options.versions && options.versions.length > 1) { @@ -200,7 +200,7 @@ // button menu var menu_button = new Ui.ButtonMenu({ icon : 'fa-caret-down', - title : 'Options', + title : (!self.workflow && 'Options') || null, tooltip : 'View available options' }); diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/mvc/tools/tools-form-workflow.js --- a/static/scripts/mvc/tools/tools-form-workflow.js +++ b/static/scripts/mvc/tools/tools-form-workflow.js @@ -9,6 +9,7 @@ initialize: function(options) { // link with node representation in workflow module this.node = workflow.active_node; + if (!this.node) { console.debug('FAILED - tools-form-workflow:initialize() - Node not found in workflow.'); return; @@ -18,6 +19,13 @@ this.workflow = true; this.options = options; + // traverse through dictionary + Utils.deepeach(options.inputs, function(item) { + if (item.type && (['data', 'data_hidden', 'hidden']).indexOf(item.type) == -1) { + item.optional = true; + } + }); + // load extension var self = this; Utils.get({ @@ -38,6 +46,7 @@ inputs[Utils.uuid()] = { label : 'Edit Step Attributes', type : 'section', + expand : this.node.annotation, inputs : [{ label : 'Annotation / Notes', name : 'annotation', @@ -160,7 +169,10 @@ type : 'boolean', value : 'false', ignore : 'false', - help : 'This action will send an email notifying you when the job is done.' + help : 'This action will send an email notifying you when the job is done.', + payload : { + 'host' : window.location.host + } },{ action : 'DeleteIntermediatesAction', label : 'Delete non-outputs', @@ -173,31 +185,51 @@ // visit input nodes and enrich by name/value pairs from server data var self = this; - function visit (inputs) { - for (var i in inputs) { - var input = inputs[i]; + function visit (head, head_list) { + head_list = head_list || []; + head_list.push(head); + for (var i in head.inputs) { + var input = head.inputs[i]; if (input.action) { + // construct identifier as expected by backend input.name = 'pja__' + output_id + '__' + input.action; if (input.argument) { input.name += '__' + input.argument; } + + // modify names of payload arguments + if (input.payload) { + for (var p_id in input.payload) { + var p = input.payload[p_id]; + input.payload[input.name + '__' + p_id] = p; + delete p; + } + } + + // access/verify existence of value var d = self.post_job_actions[input.action + output_id]; if (d) { + // mark as expanded + for (var j in head_list) { + head_list[j].expand = true; + } + + // update input field value if (input.argument) { input.value = d.action_arguments && d.action_arguments[input.argument] || input.value; } else { input.value = 'true'; } } - } else { - if (input.inputs) { - visit(input.inputs); - } + } + // continue with sub section + if (input.inputs) { + visit(input, head_list.slice(0)); } } } - visit(input_config.inputs); - + visit(input_config); + // return final configuration return input_config; }, @@ -205,6 +237,15 @@ /** Builds a new model through api call and recreates the entire form */ _buildModel: function() { + Galaxy.modal.show({ + title : 'Coming soon...', + body : 'This feature has not been implemented yet.', + buttons : { + 'Close' : function() { + Galaxy.modal.hide(); + } + } + }); }, /** Request a new model for an already created tool form and updates the form inputs diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/mvc/tools/tools-input.js --- a/static/scripts/mvc/tools/tools-input.js +++ b/static/scripts/mvc/tools/tools-input.js @@ -12,6 +12,7 @@ // link field this.field = options.field; + this.defaultvalue = options.defaultvalue; // set element this.setElement(this._template(options)); @@ -25,9 +26,17 @@ // add field element this.$field.prepend(this.field.$el); - // start with enabled optional fields + // decide wether to expand or collapse optional fields this.field.skip = false; - + if (options.optional) { + if ((this.field.validate && !this.field.validate()) || + (this.field.value && !this.field.value()) || + (this.field.value && Number(this.field.value()) == Number(this.defaultvalue)) || + (this.field.value && JSON.stringify(this.field.value()) == JSON.stringify(this.defaultvalue))) { + this.field.skip = true; + } + } + // refresh view this._refresh(); @@ -63,11 +72,12 @@ if (!this.field.skip) { this.$field.fadeIn('fast'); this.$title_optional.html('Disable'); - this.app.trigger('refresh'); } else { this.$field.hide(); this.$title_optional.html('Enable'); + this.field.value(this.defaultvalue); } + this.app.trigger('refresh'); }, /** Main Template diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/mvc/tools/tools-section.js --- a/static/scripts/mvc/tools/tools-section.js +++ b/static/scripts/mvc/tools/tools-section.js @@ -232,9 +232,9 @@ // create input field wrapper var input_element = new InputElement(this.app, { - label : input_def.title, - help : input_def.help, - field : repeat + label : input_def.title, + help : input_def.help, + field : repeat }); // displays as grouped subsection @@ -292,6 +292,11 @@ } }); + // show sub section if requested + if (input_def.expand) { + portlet.$header.trigger('click'); + } + // create table row this.table.add(portlet.$el); @@ -313,10 +318,11 @@ // create input field wrapper var input_element = new InputElement(this.app, { - label : input_def.label, - optional : input_def.optional, - help : input_def.help, - field : field + label : input_def.label, + defaultvalue : input_def.defaultvalue, + optional : input_def.optional, + help : input_def.help, + field : field }); // add to element list diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/mvc/tools/tools-template.js --- a/static/scripts/mvc/tools/tools-template.js +++ b/static/scripts/mvc/tools/tools-template.js @@ -4,7 +4,7 @@ // tool form templates return { help: function(content) { - return '<div class="toolHelp">' + + return '<div class="toolHelp" style="overflow: auto;">' + '<div class="toolHelpBody">' + content + '</div>' + diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/mvc/tools/tools-tree.js --- a/static/scripts/mvc/tools/tools-tree.js +++ b/static/scripts/mvc/tools/tools-tree.js @@ -122,13 +122,21 @@ value = patch[input.type](value); } - // handle default value + // handle simple value if (!field.skip) { - if (field.validate && !field.validate(value)) { + if (field.validate && !field.validate()) { value = null; } - if (input.ignore === undefined || (value && input.ignore != value)) { + if (input.ignore === undefined || (value !== null && input.ignore != value)) { + // add value to submission add (job_input_id, input.id, value); + + // add payload to submission + if (input.payload) { + for (var p_id in input.payload) { + add (p_id, input.id, input.payload[p_id]); + } + } } } } diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/packed/mvc/tools/tools-form-base.js --- a/static/scripts/packed/mvc/tools/tools-form-base.js +++ b/static/scripts/packed/mvc/tools/tools-form-base.js @@ -1,1 +1,1 @@ -define(["utils/utils","utils/deferred","mvc/ui/ui-portlet","mvc/ui/ui-misc","mvc/citation/citation-model","mvc/citation/citation-view","mvc/tools","mvc/tools/tools-template","mvc/tools/tools-content","mvc/tools/tools-section","mvc/tools/tools-tree"],function(g,h,f,k,i,a,d,c,e,j,b){return Backbone.View.extend({initialize:function(l){console.debug(l);var m=parent.Galaxy;if(m&&m.modal){this.modal=m.modal}else{this.modal=new k.Modal.View()}if(m&&m.currUser){this.is_admin=m.currUser.get("is_admin")}else{this.is_admin=false}this.options=l;this.container=this.options.container||"body";this.deferred=new h();this.setElement("<div/>");$(this.container).append(this.$el);this._buildForm()},reciept:function(l){$(this.container).empty();$(this.container).append(l)},highlight:function(m,n,l){var o=this.element_list[m];if(o){o.error(n||"Please verify this parameter.");if(!l){$(this.container).animate({scrollTop:o.$el.offset().top-20},500)}}},_buildForm:function(){var l=this;this.off("refresh");this.off("reset");this.field_list={};this.input_list={};this.element_list={};this.tree=new b(this);this.content=new e(this);var n=this.options;this._renderForm(n);this.tree.finalize();if(!this.workflow&&n.errors){var o=this.tree.matchResponse(n.errors);for(var m in o){this.highlight(m,o[m],true)}}this.on("refresh",function(){l.deferred.reset();l.deferred.execute(function(){l._updateModel()})});this.on("reset",function(){for(var p in this.element_list){this.element_list[p].reset()}})},_renderForm:function(u){var t=this;this.message=new k.Message();var q=new k.ButtonIcon({icon:"fa-info-circle",title:"Requirements",tooltip:"Display tool requirements",onclick:function(){if(!this.visible){this.visible=true;t.message.update({persistent:true,message:c.requirements(u),status:"info"})}else{this.visible=false;t.message.update({message:""})}}});if(!u.requirements||u.requirements.length==0){q.$el.hide()}var m=new k.ButtonMenu({icon:"fa-cubes",title:"Versions",tooltip:"Select another tool version"});if(u.versions&&u.versions.length>1){for(var o in u.versions){var r=u.versions[o];if(r!=u.version){m.addMenu({title:"Switch to "+r,version:r,icon:"fa-cube",onclick:function(){u.id=u.id.replace(u.version,this.version);u.version=this.version;t.deferred.reset();t.deferred.execute(function(){t._buildModel()})}})}}}else{m.$el.hide()}var p=new k.ButtonMenu({icon:"fa-caret-down",title:"Options",tooltip:"View available options"});if(u.biostar_url){p.addMenu({icon:"fa-question-circle",title:"Question?",tooltip:"Ask a question about this tool (Biostar)",onclick:function(){window.open(u.biostar_url+"/p/new/post/")}});p.addMenu({icon:"fa-search",title:"Search",tooltip:"Search help for this tool (Biostar)",onclick:function(){window.open(u.biostar_url+"/t/"+u.id+"/")}})}p.addMenu({icon:"fa-share",title:"Share",tooltip:"Share this tool",onclick:function(){prompt("Copy to clipboard: Ctrl+C, Enter",window.location.origin+galaxy_config.root+"root?tool_id="+u.id)}});if(this.is_admin){p.addMenu({icon:"fa-download",title:"Download",tooltip:"Download this tool",onclick:function(){window.location.href=galaxy_config.root+"api/tools/"+u.id+"/download"}})}this.section=new j.View(t,{inputs:u.inputs,cls:"ui-table-plain"});if(this.incompatible){this.$el.hide();$("#tool-form-classic").show();return}this.portlet=new f.View({icon:"fa-wrench",title:"<b>"+u.name+"</b> "+u.description+" (Galaxy Tool Version "+u.version+")",cls:"ui-portlet-slim",operations:{requirements:q,menu:p,versions:m},buttons:this.buttons});if(this.options.workflow){this.portlet.$content.css("padding","0px")}this.portlet.append(this.message.$el,true);this.portlet.append(this.section.$el);this.$el.empty();this.$el.append(this.portlet.$el);if(u.help!=""){this.$el.append(c.help(u.help))}if(u.citations){var s=$("<div/>");var l=new i.ToolCitationCollection();l.tool_id=u.id;var n=new a.CitationListView({el:s,collection:l});n.render();l.fetch();this.$el.append(s)}if(u.message){this.message.update({persistent:true,status:"warning",message:u.message})}}})}); \ No newline at end of file +define(["utils/utils","utils/deferred","mvc/ui/ui-portlet","mvc/ui/ui-misc","mvc/citation/citation-model","mvc/citation/citation-view","mvc/tools","mvc/tools/tools-template","mvc/tools/tools-content","mvc/tools/tools-section","mvc/tools/tools-tree"],function(g,h,f,k,i,a,d,c,e,j,b){return Backbone.View.extend({initialize:function(l){console.debug(l);var m=parent.Galaxy;if(m&&m.modal){this.modal=m.modal}else{this.modal=new k.Modal.View()}if(m&&m.currUser){this.is_admin=m.currUser.get("is_admin")}else{this.is_admin=false}this.options=l;this.container=this.options.container||"body";this.deferred=new h();this.setElement("<div/>");$(this.container).append(this.$el);this._buildForm()},reciept:function(l){$(this.container).empty();$(this.container).append(l)},highlight:function(m,n,l){var o=this.element_list[m];if(o){o.error(n||"Please verify this parameter.");if(!l){$(this.container).animate({scrollTop:o.$el.offset().top-20},500)}}},_buildForm:function(){var l=this;this.off("refresh");this.off("reset");this.field_list={};this.input_list={};this.element_list={};this.tree=new b(this);this.content=new e(this);var n=this.options;this._renderForm(n);this.tree.finalize();if(n.errors){var o=this.tree.matchResponse(n.errors);for(var m in o){this.highlight(m,o[m],true)}}this.on("refresh",function(){l.deferred.reset();l.deferred.execute(function(){l._updateModel()})});this.on("reset",function(){for(var p in this.element_list){this.element_list[p].reset()}})},_renderForm:function(u){var t=this;this.message=new k.Message();var q=new k.ButtonIcon({icon:"fa-info-circle",title:(!t.workflow&&"Requirements")||null,tooltip:"Display tool requirements",onclick:function(){if(!this.visible){this.visible=true;t.message.update({persistent:true,message:c.requirements(u),status:"info"})}else{this.visible=false;t.message.update({message:""})}}});if(!u.requirements||u.requirements.length==0){q.$el.hide()}var m=new k.ButtonMenu({icon:"fa-cubes",title:(!t.workflow&&"Versions")||null,tooltip:"Select another tool version"});if(u.versions&&u.versions.length>1){for(var o in u.versions){var r=u.versions[o];if(r!=u.version){m.addMenu({title:"Switch to "+r,version:r,icon:"fa-cube",onclick:function(){u.id=u.id.replace(u.version,this.version);u.version=this.version;t.deferred.reset();t.deferred.execute(function(){t._buildModel()})}})}}}else{m.$el.hide()}var p=new k.ButtonMenu({icon:"fa-caret-down",title:(!t.workflow&&"Options")||null,tooltip:"View available options"});if(u.biostar_url){p.addMenu({icon:"fa-question-circle",title:"Question?",tooltip:"Ask a question about this tool (Biostar)",onclick:function(){window.open(u.biostar_url+"/p/new/post/")}});p.addMenu({icon:"fa-search",title:"Search",tooltip:"Search help for this tool (Biostar)",onclick:function(){window.open(u.biostar_url+"/t/"+u.id+"/")}})}p.addMenu({icon:"fa-share",title:"Share",tooltip:"Share this tool",onclick:function(){prompt("Copy to clipboard: Ctrl+C, Enter",window.location.origin+galaxy_config.root+"root?tool_id="+u.id)}});if(this.is_admin){p.addMenu({icon:"fa-download",title:"Download",tooltip:"Download this tool",onclick:function(){window.location.href=galaxy_config.root+"api/tools/"+u.id+"/download"}})}this.section=new j.View(t,{inputs:u.inputs,cls:"ui-table-plain"});if(this.incompatible){this.$el.hide();$("#tool-form-classic").show();return}this.portlet=new f.View({icon:"fa-wrench",title:"<b>"+u.name+"</b> "+u.description+" (Galaxy Tool Version "+u.version+")",cls:"ui-portlet-slim",operations:{requirements:q,menu:p,versions:m},buttons:this.buttons});if(this.options.workflow){this.portlet.$content.css("padding","0px")}this.portlet.append(this.message.$el,true);this.portlet.append(this.section.$el);this.$el.empty();this.$el.append(this.portlet.$el);if(u.help!=""){this.$el.append(c.help(u.help))}if(u.citations){var s=$("<div/>");var l=new i.ToolCitationCollection();l.tool_id=u.id;var n=new a.CitationListView({el:s,collection:l});n.render();l.fetch();this.$el.append(s)}if(u.message){this.message.update({persistent:true,status:"warning",message:u.message})}}})}); \ No newline at end of file diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/packed/mvc/tools/tools-form-workflow.js --- a/static/scripts/packed/mvc/tools/tools-form-workflow.js +++ b/static/scripts/packed/mvc/tools/tools-form-workflow.js @@ -1,1 +1,1 @@ -define(["utils/utils","mvc/tools/tools-form-base"],function(b,a){var c=a.extend({initialize:function(e){this.node=workflow.active_node;if(!this.node){console.debug("FAILED - tools-form-workflow:initialize() - Node not found in workflow.");return}this.workflow=true;this.options=e;var d=this;b.get({url:galaxy_config.root+"api/datatypes",cache:true,success:function(f){d.datatypes=f;d._makeSections(e.inputs);a.prototype.initialize.call(d,e)}})},_makeSections:function(d){d[b.uuid()]={label:"Edit Step Attributes",type:"section",inputs:[{label:"Annotation / Notes",name:"annotation",type:"text",area:true,help:"Add an annotation or notes to this step; annotations are available when a workflow is viewed.",value:this.node.annotation}]};this.post_job_actions=this.node.post_job_actions;for(var e in this.node.output_terminals){d[b.uuid()]=this._makeSection(e)}},_makeSection:function(h){var g=[];for(key in this.datatypes){g.push({0:this.datatypes[key],1:this.datatypes[key]})}g.sort(function(j,i){return j.label>i.label?1:j.label<i.label?-1:0});g.unshift({0:"Sequences",1:"Sequences"});g.unshift({0:"Roadmaps",1:"Roadmaps"});g.unshift({0:"Leave unchanged",1:"None"});var f={label:"Edit Step Action: '"+h+"'",type:"section",inputs:[{action:"RenameDatasetAction",argument:"newname",label:"Rename dataset",type:"text",value:"",ignore:"",help:'This action will rename the result dataset. Click <a href="https://wiki.galaxyproject.org/Learn/AdvancedWorkflow/Variables">here</a> for more information.'},{action:"ChangeDatatypeAction",argument:"newtype",label:"Change datatype",type:"select",ignore:"None",options:g,help:"This action will change the datatype of the output to the indicated value."},{action:"TagDatasetAction",argument:"tags",label:"Tags",type:"text",value:"",ignore:"",help:"This action will set tags for the dataset."},{label:"Assign columns",type:"section",inputs:[{action:"ColumnSetAction",argument:"chromCol",label:"Chrom column",type:"text",value:"",ignore:""},{action:"ColumnSetAction",argument:"startCol",label:"Start column",type:"text",value:"",ignore:""},{action:"ColumnSetAction",argument:"endCol",label:"End column",type:"text",value:"",ignore:""},{action:"ColumnSetAction",argument:"strandCol",label:"Strand column",type:"text",value:"",ignore:""},{action:"ColumnSetAction",argument:"nameCol",label:"Name column",type:"text",value:"",ignore:""}],help:"This action will set column assignments in the output dataset. Blank fields are ignored."},{action:"EmailAction",label:"Email notification",type:"boolean",value:"false",ignore:"false",help:"This action will send an email notifying you when the job is done."},{action:"DeleteIntermediatesAction",label:"Delete non-outputs",type:"boolean",value:"false",ignore:"false",help:"All non-output steps of this workflow will have datasets deleted if they are no longer being used as job inputs when the job this action is attached to is finished. You *must* be using workflow outputs (the snowflake) in your workflow for this to have any effect."}]};var d=this;function e(j){for(var l in j){var k=j[l];if(k.action){k.name="pja__"+h+"__"+k.action;if(k.argument){k.name+="__"+k.argument}var m=d.post_job_actions[k.action+h];if(m){if(k.argument){k.value=m.action_arguments&&m.action_arguments[k.argument]||k.value}else{k.value="true"}}}else{if(k.inputs){e(k.inputs)}}}}e(f.inputs);return f},_buildModel:function(){},_updateModel:function(){var d=this;var e=this.tree.finalize();console.debug("tools-form-workflow::_refreshForm() - Refreshing states.");console.debug(e);var g=this.deferred.register();var f=galaxy_config.root+"workflow/editor_form_post?tool_id="+this.options.id;b.request({type:"GET",url:f,data:e,success:function(h){d.node.update_field_data(h,true);d.deferred.done(g);console.debug("tools-form::_refreshForm() - States refreshed.");console.debug(h)},error:function(h){d.deferred.done(g);console.debug("tools-form::_refreshForm() - Refresh request failed.");console.debug(h)}})}});return{View:c}}); \ No newline at end of file +define(["utils/utils","mvc/tools/tools-form-base"],function(b,a){var c=a.extend({initialize:function(e){this.node=workflow.active_node;if(!this.node){console.debug("FAILED - tools-form-workflow:initialize() - Node not found in workflow.");return}this.workflow=true;this.options=e;b.deepeach(e.inputs,function(f){if(f.type&&(["data","data_hidden","hidden"]).indexOf(f.type)==-1){f.optional=true}});var d=this;b.get({url:galaxy_config.root+"api/datatypes",cache:true,success:function(f){d.datatypes=f;d._makeSections(e.inputs);a.prototype.initialize.call(d,e)}})},_makeSections:function(d){d[b.uuid()]={label:"Edit Step Attributes",type:"section",expand:this.node.annotation,inputs:[{label:"Annotation / Notes",name:"annotation",type:"text",area:true,help:"Add an annotation or notes to this step; annotations are available when a workflow is viewed.",value:this.node.annotation}]};this.post_job_actions=this.node.post_job_actions;for(var e in this.node.output_terminals){d[b.uuid()]=this._makeSection(e)}},_makeSection:function(h){var g=[];for(key in this.datatypes){g.push({0:this.datatypes[key],1:this.datatypes[key]})}g.sort(function(j,i){return j.label>i.label?1:j.label<i.label?-1:0});g.unshift({0:"Sequences",1:"Sequences"});g.unshift({0:"Roadmaps",1:"Roadmaps"});g.unshift({0:"Leave unchanged",1:"None"});var f={label:"Edit Step Action: '"+h+"'",type:"section",inputs:[{action:"RenameDatasetAction",argument:"newname",label:"Rename dataset",type:"text",value:"",ignore:"",help:'This action will rename the result dataset. Click <a href="https://wiki.galaxyproject.org/Learn/AdvancedWorkflow/Variables">here</a> for more information.'},{action:"ChangeDatatypeAction",argument:"newtype",label:"Change datatype",type:"select",ignore:"None",options:g,help:"This action will change the datatype of the output to the indicated value."},{action:"TagDatasetAction",argument:"tags",label:"Tags",type:"text",value:"",ignore:"",help:"This action will set tags for the dataset."},{label:"Assign columns",type:"section",inputs:[{action:"ColumnSetAction",argument:"chromCol",label:"Chrom column",type:"text",value:"",ignore:""},{action:"ColumnSetAction",argument:"startCol",label:"Start column",type:"text",value:"",ignore:""},{action:"ColumnSetAction",argument:"endCol",label:"End column",type:"text",value:"",ignore:""},{action:"ColumnSetAction",argument:"strandCol",label:"Strand column",type:"text",value:"",ignore:""},{action:"ColumnSetAction",argument:"nameCol",label:"Name column",type:"text",value:"",ignore:""}],help:"This action will set column assignments in the output dataset. Blank fields are ignored."},{action:"EmailAction",label:"Email notification",type:"boolean",value:"false",ignore:"false",help:"This action will send an email notifying you when the job is done.",payload:{host:window.location.host}},{action:"DeleteIntermediatesAction",label:"Delete non-outputs",type:"boolean",value:"false",ignore:"false",help:"All non-output steps of this workflow will have datasets deleted if they are no longer being used as job inputs when the job this action is attached to is finished. You *must* be using workflow outputs (the snowflake) in your workflow for this to have any effect."}]};var d=this;function e(n,o){o=o||[];o.push(n);for(var m in n.inputs){var k=n.inputs[m];if(k.action){k.name="pja__"+h+"__"+k.action;if(k.argument){k.name+="__"+k.argument}if(k.payload){for(var s in k.payload){var q=k.payload[s];k.payload[k.name+"__"+s]=q;delete q}}var r=d.post_job_actions[k.action+h];if(r){for(var l in o){o[l].expand=true}if(k.argument){k.value=r.action_arguments&&r.action_arguments[k.argument]||k.value}else{k.value="true"}}}if(k.inputs){e(k,o.slice(0))}}}e(f);return f},_buildModel:function(){Galaxy.modal.show({title:"Coming soon...",body:"This feature has not been implemented yet.",buttons:{Close:function(){Galaxy.modal.hide()}}})},_updateModel:function(){var d=this;var e=this.tree.finalize();console.debug("tools-form-workflow::_refreshForm() - Refreshing states.");console.debug(e);var g=this.deferred.register();var f=galaxy_config.root+"workflow/editor_form_post?tool_id="+this.options.id;b.request({type:"GET",url:f,data:e,success:function(h){d.node.update_field_data(h,true);d.deferred.done(g);console.debug("tools-form::_refreshForm() - States refreshed.");console.debug(h)},error:function(h){d.deferred.done(g);console.debug("tools-form::_refreshForm() - Refresh request failed.");console.debug(h)}})}});return{View:c}}); \ No newline at end of file diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/packed/mvc/tools/tools-input.js --- a/static/scripts/packed/mvc/tools/tools-input.js +++ b/static/scripts/packed/mvc/tools/tools-input.js @@ -1,1 +1,1 @@ -define([],function(){return Backbone.View.extend({initialize:function(c,b){this.app=c;this.field=b.field;this.setElement(this._template(b));this.$field=this.$el.find(".ui-table-form-field");this.$title_optional=this.$el.find(".ui-table-form-title-optional");this.$error_text=this.$el.find(".ui-table-form-error-text");this.$error=this.$el.find(".ui-table-form-error");this.$field.prepend(this.field.$el);this.field.skip=false;this._refresh();var a=this;this.$title_optional.on("click",function(){a.field.skip=!a.field.skip;a._refresh()})},error:function(a){this.$error_text.html(a);this.$error.fadeIn();this.$el.addClass("ui-error")},reset:function(){this.$error.hide();this.$el.removeClass("ui-error")},_refresh:function(){if(!this.field.skip){this.$field.fadeIn("fast");this.$title_optional.html("Disable");this.app.trigger("refresh")}else{this.$field.hide();this.$title_optional.html("Enable")}},_template:function(a){var b='<div class="ui-table-form-element"><div class="ui-table-form-error ui-error"><span class="fa fa-arrow-down"/><span class="ui-table-form-error-text"/></div><div class="ui-table-form-title-strong">';if(a.optional){b+="Optional: "+a.label+'<span>&nbsp[<span class="ui-table-form-title-optional"/>]</span>'}else{b+=a.label}b+='</div><div class="ui-table-form-field">';if(a.help){b+='<div class="ui-table-form-info">'+a.help+"</div>"}b+="</div></div>";return b}})}); \ No newline at end of file +define([],function(){return Backbone.View.extend({initialize:function(c,b){this.app=c;this.field=b.field;this.defaultvalue=b.defaultvalue;this.setElement(this._template(b));this.$field=this.$el.find(".ui-table-form-field");this.$title_optional=this.$el.find(".ui-table-form-title-optional");this.$error_text=this.$el.find(".ui-table-form-error-text");this.$error=this.$el.find(".ui-table-form-error");this.$field.prepend(this.field.$el);this.field.skip=false;if(b.optional){if((this.field.validate&&!this.field.validate())||(this.field.value&&!this.field.value())||(this.field.value&&Number(this.field.value())==Number(this.defaultvalue))||(this.field.value&&JSON.stringify(this.field.value())==JSON.stringify(this.defaultvalue))){this.field.skip=true}}this._refresh();var a=this;this.$title_optional.on("click",function(){a.field.skip=!a.field.skip;a._refresh()})},error:function(a){this.$error_text.html(a);this.$error.fadeIn();this.$el.addClass("ui-error")},reset:function(){this.$error.hide();this.$el.removeClass("ui-error")},_refresh:function(){if(!this.field.skip){this.$field.fadeIn("fast");this.$title_optional.html("Disable")}else{this.$field.hide();this.$title_optional.html("Enable");this.field.value(this.defaultvalue)}this.app.trigger("refresh")},_template:function(a){var b='<div class="ui-table-form-element"><div class="ui-table-form-error ui-error"><span class="fa fa-arrow-down"/><span class="ui-table-form-error-text"/></div><div class="ui-table-form-title-strong">';if(a.optional){b+="Optional: "+a.label+'<span>&nbsp[<span class="ui-table-form-title-optional"/>]</span>'}else{b+=a.label}b+='</div><div class="ui-table-form-field">';if(a.help){b+='<div class="ui-table-form-info">'+a.help+"</div>"}b+="</div></div>";return b}})}); \ No newline at end of file diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/packed/mvc/tools/tools-section.js --- a/static/scripts/packed/mvc/tools/tools-section.js +++ b/static/scripts/packed/mvc/tools/tools-section.js @@ -1,1 +1,1 @@ -define(["utils/utils","mvc/ui/ui-table","mvc/ui/ui-misc","mvc/ui/ui-portlet","mvc/tools/tools-repeat","mvc/tools/tools-select-content","mvc/tools/tools-input"],function(e,b,h,d,c,a,f){var g=Backbone.View.extend({initialize:function(j,i){this.app=j;this.inputs=i.inputs;i.cls_tr="section-row";this.table=new b.View(i);this.setElement(this.table.$el);this.render()},render:function(){this.table.delAll();for(var j in this.inputs){this.add(this.inputs[j])}},add:function(k){var j=this;var i=jQuery.extend(true,{},k);i.id=k.id=e.uuid();this.app.input_list[i.id]=i;var l=i.type;switch(l){case"conditional":this._addConditional(i);break;case"repeat":this._addRepeat(i);break;case"section":this._addSection(i);break;default:this._addRow(i)}},_addConditional:function(j){var k=this;j.test_param.id=j.id;var n=this._addRow(j.test_param);n.options.onchange=function(w){var v=k.app.tree.matchCase(j,w);for(var u in j.cases){var q=j.cases[u];var t=j.id+"-section-"+u;var p=k.table.get(t);var s=false;for(var r in q.inputs){if(!q.inputs[r].hidden){s=true;break}}if(u==v&&s){p.fadeIn("fast")}else{p.hide()}}k.app.trigger("refresh")};for(var m in j.cases){var l=j.id+"-section-"+m;var o=new g(this.app,{inputs:j.cases[m].inputs,cls:"ui-table-plain"});o.$el.addClass("ui-table-form-section");this.table.add(o.$el);this.table.append(l)}n.trigger("change")},_addRepeat:function(p){var s=this;var q=0;function n(i,u){var t=p.id+"-section-"+(q++);var v=null;if(u){v=function(){l.del(t);l.retitle(p.title);s.app.trigger("refresh")}}var w=new g(s.app,{inputs:i,cls:"ui-table-plain"});l.add({id:t,title:p.title,$el:w.$el,ondel:v});l.retitle(p.title)}var l=new c.View({title_new:p.title,max:p.max,onnew:function(){n(p.inputs,true);s.app.trigger("refresh")}});var j=p.min;var r=_.size(p.cache);for(var m=0;m<Math.max(r,j);m++){var o=null;if(m<r){o=p.cache[m]}else{o=p.inputs}n(o,m>=j)}var k=new f(this.app,{label:p.title,help:p.help,field:l});k.$el.addClass("ui-table-form-section");this.table.add(k.$el);this.table.append(p.id)},_addSection:function(i){var j=this;var n=new g(j.app,{inputs:i.inputs,cls:"ui-table-plain"});var m=new h.ButtonIcon({icon:"fa-eye-slash",tooltip:"Show/hide section",cls:"ui-button-icon-plain"});var l=new d.View({title:i.label,cls:"ui-portlet-section",operations:{button_visible:m}});l.append(n.$el);var k=false;l.$content.hide();l.$header.css("cursor","pointer");l.$header.on("click",function(){if(k){k=false;l.$content.hide();m.setIcon("fa-eye-slash")}else{k=true;l.$content.fadeIn("fast");m.setIcon("fa-eye")}});this.table.add(l.$el);this.table.append(i.id)},_addRow:function(i){var l=i.id;var j=this._createField(i);this.app.field_list[l]=j;var k=new f(this.app,{label:i.label,optional:i.optional,help:i.help,field:j});this.app.element_list[l]=k;this.table.add(k.$el);this.table.append(l);if(i.hidden){this.table.get(l).hide()}return j},_createField:function(i){var j=null;switch(i.type){case"text":j=this._fieldText(i);break;case"select":j=this._fieldSelect(i);break;case"data":j=this._fieldData(i);break;case"data_collection":j=this._fieldData(i);break;case"data_column":i.error_text="Missing columns in referenced dataset.";j=this._fieldSelect(i);break;case"hidden":j=this._fieldHidden(i);break;case"hidden_data":j=this._fieldHidden(i);break;case"integer":j=this._fieldSlider(i);break;case"float":j=this._fieldSlider(i);break;case"boolean":j=this._fieldBoolean(i);break;case"genomebuild":i.searchable=true;j=this._fieldSelect(i);break;case"drill_down":j=this._fieldDrilldown(i);break;case"baseurl":j=this._fieldHidden(i);break;default:this.app.incompatible=true;if(i.options){j=this._fieldSelect(i)}else{j=this._fieldText(i)}console.debug("tools-form::_addRow() : Auto matched field type ("+i.type+").")}if(i.value!==undefined){j.value(i.value)}return j},_fieldData:function(i){if(this.app.workflow){var k=e.textify(i.extensions.toString());i.info="Data input '"+i.name+"' ("+k+")";return this._fieldHidden(i)}var j=this;return new a.View(this.app,{id:"field-"+i.id,extensions:i.extensions,multiple:i.multiple,type:i.type,data:i.options,onchange:function(){j.app.trigger("refresh")}})},_fieldSelect:function(j){if(this.app.workflow&&j.is_dynamic){if(!e.validate(j.value)){j.value=""}return this._fieldText(j)}var l=[];for(var m in j.options){var n=j.options[m];l.push({label:n[0],value:n[1]})}var o=h.Select;switch(j.display){case"checkboxes":o=h.Checkbox;break;case"radio":o=h.Radio;break}var k=this;return new o.View({id:"field-"+j.id,data:l,error_text:j.error_text||"No options available",multiple:j.multiple,searchable:j.searchable,onchange:function(){k.app.trigger("refresh")}})},_fieldDrilldown:function(i){var j=this;return new h.Drilldown.View({id:"field-"+i.id,data:i.options,display:i.display,onchange:function(){j.app.trigger("refresh")}})},_fieldText:function(i){var j=this;return new h.Input({id:"field-"+i.id,area:i.area,onchange:function(){j.app.trigger("refresh")}})},_fieldSlider:function(i){var j=this;return new h.Slider.View({id:"field-"+i.id,precise:i.type=="float",min:i.min,max:i.max,onchange:function(){j.app.trigger("refresh")}})},_fieldHidden:function(i){return new h.Hidden({id:"field-"+i.id,info:i.info})},_fieldBoolean:function(i){var j=this;return new h.RadioButton.View({id:"field-"+i.id,data:[{label:"Yes",value:"true"},{label:"No",value:"false"}],onchange:function(){j.app.trigger("refresh")}})}});return{View:g}}); \ No newline at end of file +define(["utils/utils","mvc/ui/ui-table","mvc/ui/ui-misc","mvc/ui/ui-portlet","mvc/tools/tools-repeat","mvc/tools/tools-select-content","mvc/tools/tools-input"],function(e,b,h,d,c,a,f){var g=Backbone.View.extend({initialize:function(j,i){this.app=j;this.inputs=i.inputs;i.cls_tr="section-row";this.table=new b.View(i);this.setElement(this.table.$el);this.render()},render:function(){this.table.delAll();for(var j in this.inputs){this.add(this.inputs[j])}},add:function(k){var j=this;var i=jQuery.extend(true,{},k);i.id=k.id=e.uuid();this.app.input_list[i.id]=i;var l=i.type;switch(l){case"conditional":this._addConditional(i);break;case"repeat":this._addRepeat(i);break;case"section":this._addSection(i);break;default:this._addRow(i)}},_addConditional:function(j){var k=this;j.test_param.id=j.id;var n=this._addRow(j.test_param);n.options.onchange=function(w){var v=k.app.tree.matchCase(j,w);for(var u in j.cases){var q=j.cases[u];var t=j.id+"-section-"+u;var p=k.table.get(t);var s=false;for(var r in q.inputs){if(!q.inputs[r].hidden){s=true;break}}if(u==v&&s){p.fadeIn("fast")}else{p.hide()}}k.app.trigger("refresh")};for(var m in j.cases){var l=j.id+"-section-"+m;var o=new g(this.app,{inputs:j.cases[m].inputs,cls:"ui-table-plain"});o.$el.addClass("ui-table-form-section");this.table.add(o.$el);this.table.append(l)}n.trigger("change")},_addRepeat:function(p){var s=this;var q=0;function n(i,u){var t=p.id+"-section-"+(q++);var v=null;if(u){v=function(){l.del(t);l.retitle(p.title);s.app.trigger("refresh")}}var w=new g(s.app,{inputs:i,cls:"ui-table-plain"});l.add({id:t,title:p.title,$el:w.$el,ondel:v});l.retitle(p.title)}var l=new c.View({title_new:p.title,max:p.max,onnew:function(){n(p.inputs,true);s.app.trigger("refresh")}});var j=p.min;var r=_.size(p.cache);for(var m=0;m<Math.max(r,j);m++){var o=null;if(m<r){o=p.cache[m]}else{o=p.inputs}n(o,m>=j)}var k=new f(this.app,{label:p.title,help:p.help,field:l});k.$el.addClass("ui-table-form-section");this.table.add(k.$el);this.table.append(p.id)},_addSection:function(i){var j=this;var n=new g(j.app,{inputs:i.inputs,cls:"ui-table-plain"});var m=new h.ButtonIcon({icon:"fa-eye-slash",tooltip:"Show/hide section",cls:"ui-button-icon-plain"});var l=new d.View({title:i.label,cls:"ui-portlet-section",operations:{button_visible:m}});l.append(n.$el);var k=false;l.$content.hide();l.$header.css("cursor","pointer");l.$header.on("click",function(){if(k){k=false;l.$content.hide();m.setIcon("fa-eye-slash")}else{k=true;l.$content.fadeIn("fast");m.setIcon("fa-eye")}});if(i.expand){l.$header.trigger("click")}this.table.add(l.$el);this.table.append(i.id)},_addRow:function(i){var l=i.id;var j=this._createField(i);this.app.field_list[l]=j;var k=new f(this.app,{label:i.label,defaultvalue:i.defaultvalue,optional:i.optional,help:i.help,field:j});this.app.element_list[l]=k;this.table.add(k.$el);this.table.append(l);if(i.hidden){this.table.get(l).hide()}return j},_createField:function(i){var j=null;switch(i.type){case"text":j=this._fieldText(i);break;case"select":j=this._fieldSelect(i);break;case"data":j=this._fieldData(i);break;case"data_collection":j=this._fieldData(i);break;case"data_column":i.error_text="Missing columns in referenced dataset.";j=this._fieldSelect(i);break;case"hidden":j=this._fieldHidden(i);break;case"hidden_data":j=this._fieldHidden(i);break;case"integer":j=this._fieldSlider(i);break;case"float":j=this._fieldSlider(i);break;case"boolean":j=this._fieldBoolean(i);break;case"genomebuild":i.searchable=true;j=this._fieldSelect(i);break;case"drill_down":j=this._fieldDrilldown(i);break;case"baseurl":j=this._fieldHidden(i);break;default:this.app.incompatible=true;if(i.options){j=this._fieldSelect(i)}else{j=this._fieldText(i)}console.debug("tools-form::_addRow() : Auto matched field type ("+i.type+").")}if(i.value!==undefined){j.value(i.value)}return j},_fieldData:function(i){if(this.app.workflow){var k=e.textify(i.extensions.toString());i.info="Data input '"+i.name+"' ("+k+")";return this._fieldHidden(i)}var j=this;return new a.View(this.app,{id:"field-"+i.id,extensions:i.extensions,multiple:i.multiple,type:i.type,data:i.options,onchange:function(){j.app.trigger("refresh")}})},_fieldSelect:function(j){if(this.app.workflow&&j.is_dynamic){if(!e.validate(j.value)){j.value=""}return this._fieldText(j)}var l=[];for(var m in j.options){var n=j.options[m];l.push({label:n[0],value:n[1]})}var o=h.Select;switch(j.display){case"checkboxes":o=h.Checkbox;break;case"radio":o=h.Radio;break}var k=this;return new o.View({id:"field-"+j.id,data:l,error_text:j.error_text||"No options available",multiple:j.multiple,searchable:j.searchable,onchange:function(){k.app.trigger("refresh")}})},_fieldDrilldown:function(i){var j=this;return new h.Drilldown.View({id:"field-"+i.id,data:i.options,display:i.display,onchange:function(){j.app.trigger("refresh")}})},_fieldText:function(i){var j=this;return new h.Input({id:"field-"+i.id,area:i.area,onchange:function(){j.app.trigger("refresh")}})},_fieldSlider:function(i){var j=this;return new h.Slider.View({id:"field-"+i.id,precise:i.type=="float",min:i.min,max:i.max,onchange:function(){j.app.trigger("refresh")}})},_fieldHidden:function(i){return new h.Hidden({id:"field-"+i.id,info:i.info})},_fieldBoolean:function(i){var j=this;return new h.RadioButton.View({id:"field-"+i.id,data:[{label:"Yes",value:"true"},{label:"No",value:"false"}],onchange:function(){j.app.trigger("refresh")}})}});return{View:g}}); \ No newline at end of file diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/packed/mvc/tools/tools-template.js --- a/static/scripts/packed/mvc/tools/tools-template.js +++ b/static/scripts/packed/mvc/tools/tools-template.js @@ -1,1 +1,1 @@ -define([],function(){return{help:function(a){return'<div class="toolHelp"><div class="toolHelpBody">'+a+"</div></div>"},success:function(c){if(!c.jobs||!c.jobs.length){console.debug("tools-template::success() - Failed jobs.");return}var a=c.jobs.length;var d="";if(a==1){d="1 job has"}else{d=a+" jobs have"}var b='<div class="donemessagelarge"><p>'+d+" been successfully added to the queue - resulting in the following datasets:</p>";for(var e in c.outputs){b+='<p style="padding: 10px 20px;"><b>'+(parseInt(e)+1)+": "+c.outputs[e].name+"</b></p>"}b+="<p>You can check the status of queued jobs and view the resulting data by refreshing the History pane. When the job has been run the status will change from 'running' to 'finished' if completed successfully or 'error' if problems were encountered.</p></div>";return b},error:function(a){return'<div><p>The server could not complete the request. Please contact the Galaxy Team if this error persists.</p><textarea class="ui-textarea" disabled style="color: black;" rows="6">'+JSON.stringify(a,undefined,4)+"</textarea></div>"},batchMode:function(){return'<div class="ui-table-form-info"><i class="fa fa-sitemap" style="font-size: 1.2em; padding: 2px 5px;"/>This is a batch mode input field. A separate job will be triggered for each dataset.</div>'},requirements:function(a){var d="This tool requires ";for(var b in a.requirements){var c=a.requirements[b];d+=c.name;if(c.version){d+=" (Version "+c.version+")"}if(b<a.requirements.length-2){d+=", "}if(b==a.requirements.length-2){d+=" and "}}return d+'. Click <a target="_blank" href="https://wiki.galaxyproject.org/Tools/Requirements">here</a> for more information.'}}}); \ No newline at end of file +define([],function(){return{help:function(a){return'<div class="toolHelp" style="overflow: auto;"><div class="toolHelpBody">'+a+"</div></div>"},success:function(c){if(!c.jobs||!c.jobs.length){console.debug("tools-template::success() - Failed jobs.");return}var a=c.jobs.length;var d="";if(a==1){d="1 job has"}else{d=a+" jobs have"}var b='<div class="donemessagelarge"><p>'+d+" been successfully added to the queue - resulting in the following datasets:</p>";for(var e in c.outputs){b+='<p style="padding: 10px 20px;"><b>'+(parseInt(e)+1)+": "+c.outputs[e].name+"</b></p>"}b+="<p>You can check the status of queued jobs and view the resulting data by refreshing the History pane. When the job has been run the status will change from 'running' to 'finished' if completed successfully or 'error' if problems were encountered.</p></div>";return b},error:function(a){return'<div><p>The server could not complete the request. Please contact the Galaxy Team if this error persists.</p><textarea class="ui-textarea" disabled style="color: black;" rows="6">'+JSON.stringify(a,undefined,4)+"</textarea></div>"},batchMode:function(){return'<div class="ui-table-form-info"><i class="fa fa-sitemap" style="font-size: 1.2em; padding: 2px 5px;"/>This is a batch mode input field. A separate job will be triggered for each dataset.</div>'},requirements:function(a){var d="This tool requires ";for(var b in a.requirements){var c=a.requirements[b];d+=c.name;if(c.version){d+=" (Version "+c.version+")"}if(b<a.requirements.length-2){d+=", "}if(b==a.requirements.length-2){d+=" and "}}return d+'. Click <a target="_blank" href="https://wiki.galaxyproject.org/Tools/Requirements">here</a> for more information.'}}}); \ No newline at end of file diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/packed/mvc/tools/tools-tree.js --- a/static/scripts/packed/mvc/tools/tools-tree.js +++ b/static/scripts/packed/mvc/tools/tools-tree.js @@ -1,1 +1,1 @@ -define(["utils/utils"],function(a){return Backbone.Model.extend({initialize:function(b){this.app=b},finalize:function(g){var b=this;this.map_dict={};if(!this.app.section){return{}}g=g||{};var f={};var e={};this._iterate(this.app.section.$el,e);function d(j,i,h){f[j]=h;b.map_dict[j]=i}function c(o,r){for(var m in r){var k=r[m];if(k.input){var t=k.input;var n=o;if(o!=""){n+="|"}n+=t.name;switch(t.type){case"repeat":var j="section-";var w=[];var q=null;for(var v in k){var p=v.indexOf(j);if(p!=-1){p+=j.length;w.push(parseInt(v.substr(p)));if(!q){q=v.substr(0,p)}}}w.sort(function(x,i){return x-i});var m=0;for(var l in w){c(n+"_"+m++,k[q+w[l]])}break;case"conditional":var u=b.app.field_list[t.id].value();if(g[t.test_param.type]){u=g[t.test_param.type](u)}d(n+"|"+t.test_param.name,t.id,u);var h=b.matchCase(t,u);if(h!=-1){c(n,r[t.id+"-section-"+h])}break;case"section":c("",k);break;default:var s=b.app.field_list[t.id];if(s&&s.value){var u=s.value();if(g[t.type]){u=g[t.type](u)}if(!s.skip){if(s.validate&&!s.validate(u)){u=null}if(t.ignore===undefined||(u&&t.ignore!=u)){d(n,t.id,u)}}}}}}}c("",e);return f},match:function(b){return this.map_dict&&this.map_dict[b]},matchCase:function(b,d){if(b.test_param.type=="boolean"){if(d=="true"){d=b.test_param.truevalue||"true"}else{d=b.test_param.falsevalue||"false"}}for(var c in b.cases){if(b.cases[c].value==d){return c}}return -1},matchModel:function(d,f){var b={};var c=this;function e(g,p){for(var m in p){var k=p[m];var n=k.name;if(g!=""){n=g+"|"+n}switch(k.type){case"repeat":for(var l in k.cache){e(n+"_"+l,k.cache[l])}break;case"conditional":var q=k.test_param&&k.test_param.value;var h=c.matchCase(k,q);if(h!=-1){e(n,k.cases[h].inputs)}break;default:var o=c.map_dict[n];if(o){f(o,k)}}}}e("",d.inputs);return b},matchResponse:function(d){var b={};var c=this;function e(l,j){if(typeof j==="string"){var g=c.map_dict[l];if(g){b[g]=j}}else{for(var h in j){var f=h;if(l!==""){var k="|";if(j instanceof Array){k="_"}f=l+k+f}e(f,j[h])}}}e("",d);return b},_iterate:function(d,e){var b=this;var c=$(d).children();c.each(function(){var h=this;var g=$(h).attr("id");if($(h).hasClass("section-row")){e[g]={};var f=b.app.input_list[g];if(f){e[g]={input:f}}b._iterate(h,e[g])}else{b._iterate(h,e)}})}})}); \ No newline at end of file +define(["utils/utils"],function(a){return Backbone.Model.extend({initialize:function(b){this.app=b},finalize:function(g){var b=this;this.map_dict={};if(!this.app.section){return{}}g=g||{};var f={};var e={};this._iterate(this.app.section.$el,e);function d(j,i,h){f[j]=h;b.map_dict[j]=i}function c(p,s){for(var n in s){var k=s[n];if(k.input){var u=k.input;var o=p;if(p!=""){o+="|"}o+=u.name;switch(u.type){case"repeat":var j="section-";var x=[];var r=null;for(var w in k){var q=w.indexOf(j);if(q!=-1){q+=j.length;x.push(parseInt(w.substr(q)));if(!r){r=w.substr(0,q)}}}x.sort(function(y,i){return y-i});var n=0;for(var l in x){c(o+"_"+n++,k[r+x[l]])}break;case"conditional":var v=b.app.field_list[u.id].value();if(g[u.test_param.type]){v=g[u.test_param.type](v)}d(o+"|"+u.test_param.name,u.id,v);var h=b.matchCase(u,v);if(h!=-1){c(o,s[u.id+"-section-"+h])}break;case"section":c("",k);break;default:var t=b.app.field_list[u.id];if(t&&t.value){var v=t.value();if(g[u.type]){v=g[u.type](v)}if(!t.skip){if(t.validate&&!t.validate()){v=null}if(u.ignore===undefined||(v!==null&&u.ignore!=v)){d(o,u.id,v);if(u.payload){for(var m in u.payload){d(m,u.id,u.payload[m])}}}}}}}}}c("",e);return f},match:function(b){return this.map_dict&&this.map_dict[b]},matchCase:function(b,d){if(b.test_param.type=="boolean"){if(d=="true"){d=b.test_param.truevalue||"true"}else{d=b.test_param.falsevalue||"false"}}for(var c in b.cases){if(b.cases[c].value==d){return c}}return -1},matchModel:function(d,f){var b={};var c=this;function e(g,p){for(var m in p){var k=p[m];var n=k.name;if(g!=""){n=g+"|"+n}switch(k.type){case"repeat":for(var l in k.cache){e(n+"_"+l,k.cache[l])}break;case"conditional":var q=k.test_param&&k.test_param.value;var h=c.matchCase(k,q);if(h!=-1){e(n,k.cases[h].inputs)}break;default:var o=c.map_dict[n];if(o){f(o,k)}}}}e("",d.inputs);return b},matchResponse:function(d){var b={};var c=this;function e(l,j){if(typeof j==="string"){var g=c.map_dict[l];if(g){b[g]=j}}else{for(var h in j){var f=h;if(l!==""){var k="|";if(j instanceof Array){k="_"}f=l+k+f}e(f,j[h])}}}e("",d);return b},_iterate:function(d,e){var b=this;var c=$(d).children();c.each(function(){var h=this;var g=$(h).attr("id");if($(h).hasClass("section-row")){e[g]={};var f=b.app.input_list[g];if(f){e[g]={input:f}}b._iterate(h,e[g])}else{b._iterate(h,e)}})}})}); \ No newline at end of file diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/packed/utils/utils.js --- a/static/scripts/packed/utils/utils.js +++ b/static/scripts/packed/utils/utils.js @@ -1,1 +1,1 @@ -define(["libs/underscore"],function(m){function d(n){return $("<div/>").text(n).html()}function l(o){if(!(o instanceof Array)){o=[o]}for(var n in o){if(["None",null,"null",undefined,"undefined"].indexOf(o[n])>-1){return false}}return true}function h(n){var n=n.toString();if(n){n=n.replace(/,/g,", ");var o=n.lastIndexOf(", ");if(o!=-1){n=n.substr(0,o)+" or "+n.substr(o+1)}return n}return""}function e(n){top.__utils__get__=top.__utils__get__||{};if(n.cache&&top.__utils__get__[n.url]){n.success&&n.success(top.__utils__get__[n.url]);console.debug("utils.js::get() - Fetching from cache ["+n.url+"].")}else{i({url:n.url,data:n.data,success:function(o){top.__utils__get__[n.url]=o;n.success&&n.success(o)},error:function(o){n.error&&n.error(o)}})}}function i(o){var n={contentType:"application/json",type:o.type||"GET",data:o.data||{},url:o.url};if(n.type=="GET"||n.type=="DELETE"){if(n.url.indexOf("?")==-1){n.url+="?"}else{n.url+="&"}n.url=n.url+$.param(n.data);n.data=null}else{n.dataType="json";n.url=n.url;n.data=JSON.stringify(n.data)}$.ajax(n).done(function(p){if(typeof p==="string"){try{p=p.replace("Infinity,",'"Infinity",');p=jQuery.parseJSON(p)}catch(q){console.debug(q)}}o.success&&o.success(p)}).fail(function(q){var p=null;try{p=jQuery.parseJSON(q.responseText)}catch(r){p=q.responseText}o.error&&o.error(p,q)})}function j(q,n){var o=$('<div class="'+q+'"></div>');o.appendTo(":eq(0)");var p=o.css(n);o.remove();return p}function g(n){if(!$('link[href^="'+n+'"]').length){$('<link href="'+galaxy_config.root+n+'" rel="stylesheet">').appendTo("head")}}function k(n,o){if(n){return m.defaults(n,o)}else{return o}}function b(o,q){var p="";if(o>=100000000000){o=o/100000000000;p="TB"}else{if(o>=100000000){o=o/100000000;p="GB"}else{if(o>=100000){o=o/100000;p="MB"}else{if(o>=100){o=o/100;p="KB"}else{if(o>0){o=o*10;p="b"}else{return"<strong>-</strong>"}}}}}var n=(Math.round(o)/10);if(q){return n+" "+p}else{return"<strong>"+n+"</strong> "+p}}function a(){return"x"+Math.random().toString(36).substring(2,9)}function c(n){var o=$("<p></p>");o.append(n);return o}function f(){var p=new Date();var n=(p.getHours()<10?"0":"")+p.getHours();var o=(p.getMinutes()<10?"0":"")+p.getMinutes();var q=p.getDate()+"/"+(p.getMonth()+1)+"/"+p.getFullYear()+", "+n+":"+o;return q}return{cssLoadFile:g,cssGetAttribute:j,get:e,merge:k,bytesToString:b,uuid:a,time:f,wrap:c,request:i,sanitize:d,textify:h,validate:l}}); \ No newline at end of file +define(["libs/underscore"],function(m){function n(r,q){for(var o in r){var p=r[o];if(p&&typeof(p)=="object"){q(p);n(p,q)}}}function d(o){return $("<div/>").text(o).html()}function l(p){if(!(p instanceof Array)){p=[p]}for(var o in p){if(["None",null,"null",undefined,"undefined"].indexOf(p[o])>-1){return false}}return true}function h(o){var o=o.toString();if(o){o=o.replace(/,/g,", ");var p=o.lastIndexOf(", ");if(p!=-1){o=o.substr(0,p)+" or "+o.substr(p+1)}return o}return""}function e(o){top.__utils__get__=top.__utils__get__||{};if(o.cache&&top.__utils__get__[o.url]){o.success&&o.success(top.__utils__get__[o.url]);console.debug("utils.js::get() - Fetching from cache ["+o.url+"].")}else{i({url:o.url,data:o.data,success:function(p){top.__utils__get__[o.url]=p;o.success&&o.success(p)},error:function(p){o.error&&o.error(p)}})}}function i(p){var o={contentType:"application/json",type:p.type||"GET",data:p.data||{},url:p.url};if(o.type=="GET"||o.type=="DELETE"){if(o.url.indexOf("?")==-1){o.url+="?"}else{o.url+="&"}o.url=o.url+$.param(o.data);o.data=null}else{o.dataType="json";o.url=o.url;o.data=JSON.stringify(o.data)}$.ajax(o).done(function(q){if(typeof q==="string"){try{q=q.replace("Infinity,",'"Infinity",');q=jQuery.parseJSON(q)}catch(r){console.debug(r)}}p.success&&p.success(q)}).fail(function(r){var q=null;try{q=jQuery.parseJSON(r.responseText)}catch(s){q=r.responseText}p.error&&p.error(q,r)})}function j(r,o){var p=$('<div class="'+r+'"></div>');p.appendTo(":eq(0)");var q=p.css(o);p.remove();return q}function g(o){if(!$('link[href^="'+o+'"]').length){$('<link href="'+galaxy_config.root+o+'" rel="stylesheet">').appendTo("head")}}function k(o,p){if(o){return m.defaults(o,p)}else{return p}}function b(p,r){var q="";if(p>=100000000000){p=p/100000000000;q="TB"}else{if(p>=100000000){p=p/100000000;q="GB"}else{if(p>=100000){p=p/100000;q="MB"}else{if(p>=100){p=p/100;q="KB"}else{if(p>0){p=p*10;q="b"}else{return"<strong>-</strong>"}}}}}var o=(Math.round(p)/10);if(r){return o+" "+q}else{return"<strong>"+o+"</strong> "+q}}function a(){return"x"+Math.random().toString(36).substring(2,9)}function c(o){var p=$("<p></p>");p.append(o);return p}function f(){var q=new Date();var o=(q.getHours()<10?"0":"")+q.getHours();var p=(q.getMinutes()<10?"0":"")+q.getMinutes();var r=q.getDate()+"/"+(q.getMonth()+1)+"/"+q.getFullYear()+", "+o+":"+p;return r}return{cssLoadFile:g,cssGetAttribute:j,get:e,merge:k,bytesToString:b,uuid:a,time:f,wrap:c,request:i,sanitize:d,textify:h,validate:l,deepeach:n}}); \ No newline at end of file diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/scripts/utils/utils.js --- a/static/scripts/utils/utils.js +++ b/static/scripts/utils/utils.js @@ -6,6 +6,18 @@ // dependencies define(["libs/underscore"], function(_) { +/** Traverse through json +*/ +function deepeach(dict, callback) { + for (var i in dict) { + var d = dict[i]; + if (d && typeof(d) == "object") { + callback(d); + deepeach(d, callback); + } + } +} + /** * Sanitize/escape a string * @param{String} content - Content to be sanitized @@ -248,7 +260,8 @@ request: request, sanitize: sanitize, textify: textify, - validate: validate + validate: validate, + deepeach: deepeach }; }); diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/style/blue/base.css --- a/static/style/blue/base.css +++ b/static/style/blue/base.css @@ -1412,7 +1412,7 @@ .ui-button-icon-plain{border:none !important;background:none !important;height:inherit !important;width:inherit !important;padding-right:3px !important} .ui-tabs .ui-tabs-add{font-size:0.8em;margin-right:5px} .ui-tabs .ui-tabs-delete{font-size:0.8em;margin-left:5px;cursor:pointer} -.ui-portlet,.ui-portlet-slim,.ui-portlet-repeat,.ui-portlet-section,.ui-portlet-slim,.ui-portlet-repeat,.ui-portlet-section{border:solid #d6b161 1px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;position:relative;clear:both;width:auto;height:100%}.ui-portlet .portlet-header,.ui-portlet-slim .portlet-header{background:#ebd9b2;border-bottom:solid #d6b161 1px;padding:2px 8px;overflow:visible;float:right;width:100%}.ui-portlet .portlet-header .portlet-title,.ui-portlet-slim .portlet-header .portlet-title{display:inline;vertical-align:middle}.ui-portlet .portlet-header .portlet-title .portlet-title-text,.ui-portlet-slim .portlet-header .portlet-title .portlet-title-text{vertical-align:middle} +.ui-portlet,.ui-portlet-slim,.ui-portlet-repeat,.ui-portlet-section,.ui-portlet-slim,.ui-portlet-repeat,.ui-portlet-section{border:solid #d6b161 1px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;position:relative;clear:both;width:auto;height:100%}.ui-portlet .portlet-header,.ui-portlet-slim .portlet-header{background:#ebd9b2;border-bottom:solid #d6b161 1px;padding:2px 8px;overflow:visible;float:right;width:100%}.ui-portlet .portlet-header .portlet-title,.ui-portlet-slim .portlet-header .portlet-title{display:inline;vertical-align:middle}.ui-portlet .portlet-header .portlet-title .portlet-title-text,.ui-portlet-slim .portlet-header .portlet-title .portlet-title-text{vertical-align:middle;line-height:20px} .ui-portlet .portlet-header .portlet-title .icon,.ui-portlet-slim .portlet-header .portlet-title .icon{font-size:1.2em;vertical-align:middle} .ui-portlet .portlet-buttons,.ui-portlet-slim .portlet-buttons{height:50px;padding:10px} .ui-portlet .portlet-content,.ui-portlet-slim .portlet-content{height:inherit;padding:10px;clear:both}.ui-portlet .portlet-content .content,.ui-portlet-slim .portlet-content .content{padding:10px;height:100%;width:100%}.ui-portlet .portlet-content .content .buttons,.ui-portlet-slim .portlet-content .content .buttons{height:50px;padding:10px} @@ -1765,7 +1765,7 @@ div.toolFormDisabled{border-color:#bfbfbf} div.toolHelp{margin-top:15px;padding:5px} div.toolHelpBody{width:100%} -.toolForm.toolFormInCanvas{border:solid #d6b161 1px;background:#fff}.toolForm.toolFormInCanvas.toolForm-active{border:solid blue 3px;margin:4px} +.toolForm.toolFormInCanvas{border:solid #d6b161 1px;background:#fff}.toolForm.toolFormInCanvas.toolForm-active{border:solid #5f6990 3px;margin:4px} .toolForm.toolFormInCanvas .toolFormTitle{font-size:12px;line-height:1.428571429} div.form,div.toolForm{border:solid #d6b161 1px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px} div.form-title,div.toolFormTitle{padding:5px 10px;background:#ebd9b2;border-bottom:solid #d6b161 1px} diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/style/src/less/base.less --- a/static/style/src/less/base.less +++ b/static/style/src/less/base.less @@ -586,7 +586,7 @@ border: solid @form-border 1px; background: @white; &.toolForm-active { - border: solid blue 3px; + border: solid @brand-primary 3px; margin: 4px; } .toolFormTitle { diff -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 -r bf6b51623d29b68ba216d97d99bb1ea200a3e4fc static/style/src/less/ui.less --- a/static/style/src/less/ui.less +++ b/static/style/src/less/ui.less @@ -168,6 +168,7 @@ vertical-align: middle; .portlet-title-text { vertical-align: middle; + line-height: 20px; } .icon { font-size: 1.2em; Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] f005c3: Update Test
by GitHub 13 Jan '15

13 Jan '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: f005c354d24102dfe13c1337485b4c7cf4f46ecd https://github.com/galaxyproject/usegalaxy-playbook/commit/f005c354d24102df… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2015-01-13 (Tue, 13 Jan 2015) Changed paths: M files/galaxy/test.galaxyproject.org/var/integrated_tool_panel.xml M stage/group_vars/all.yml Log Message: ----------- Update Test

1 0

commit/galaxy-central: natefoo: Merge stable to default
by commits-noreply＠bitbucket.org 13 Jan '15

13 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/659fc6822f0a/ Changeset: 659fc6822f0a User: natefoo Date: 2015-01-14 03:34:26+00:00 Summary: Merge stable to default Affected #: 10 files diff -r a74b0112995cd2ae396d0916a6e845b7709f9002 -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 .hgtags --- a/.hgtags +++ b/.hgtags @@ -8,16 +8,18 @@ 5e605ed6069fe4c5ca9875e95e91b2713499e8ca release_2014.02.10 9e53251b0b7e93b9563008a2b112f2e815a04bbc release_2014.04.14 7e257c7b10badb65772b1528cb61d58175a42e47 release_2014.06.02 -7a4d321c0e38fa263ea83d29a35a608c3181fcba latest_2014.06.02 -9661b9d5d5b330483ae3ad2236410e0efaa7c500 latest_2014.04.14 -6b0bd93038a843b1585155f0d63f0eea2459c70b latest_2013.01.13 -3e62060b14b9afc46f8e0ec02e1a4500d77db9e1 latest_2013.02.08 -425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 latest_2013.04.01 -9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 latest_2013.06.03 -9ed84cd208e07e8985ec917cb025fcbbb09edcfb latest_2013.08.12 -81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 latest_2013.11.04 -2a756ca2cb1826db7796018e77d12e2dd7b67603 latest_2014.02.10 +4145417a6e1c13f82a3de365aadef0fb3ed7ab14 latest_2014.06.02 +8f9dcac033694e4cabcf5daae5cca1cfefbe967f latest_2014.04.14 +9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f latest_2013.01.13 +b986c184be88947b5d1d90be7f36cfd2627dd938 latest_2013.02.08 +dec9431d66b837a208e2f060d90afd913c721227 latest_2013.04.01 +19e56e66b0b344c6e2afa4541f6988e4fdb9af29 latest_2013.06.03 +cee903b8b3eee9145627ee89742555dac581791e latest_2013.08.12 +7d5aa19a166cba9039e15f338a1e3fc924c43d3a latest_2013.11.04 +0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 latest_2014.02.10 ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 -548ab24667d6206780237bd807f7d857a484c461 latest_2014.08.11 +8150024c0e6fc5aef3033cf8aaa574896f6b5d0d latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 -5834b1066462dd219f67c1c3cbd77c78e7cf3a6c latest_2014.10.06 +c437b28348a9345db8433e5b4f0e05ec8fb6c38a latest_2014.10.06 +2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee release_2015.01.13 +2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee latest_2015.01.13 diff -r a74b0112995cd2ae396d0916a6e845b7709f9002 -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -20,6 +20,7 @@ import galaxy.model from galaxy.util import listify +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util import stringify_dictionary_keys from galaxy.util import string_as_bool from galaxy.util import in_directory @@ -232,6 +233,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -480,6 +484,10 @@ def to_string( self, value ): return json.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return json.safe_dumps( value ) + class PythonObjectParameter( MetadataParameter ): @@ -510,6 +518,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context=None, other_values=None, **kwd ): context = context or {} other_values = other_values or {} diff -r a74b0112995cd2ae396d0916a6e845b7709f9002 -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -2,10 +2,12 @@ import tempfile from galaxy import model +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.bunch import Bunch from galaxy.util.none_like import NoneDataset from galaxy.util.template import fill_template from galaxy.tools.wrappers import ( + ToolParameterValueWrapper, DatasetFilenameWrapper, DatasetListWrapper, DatasetCollectionWrapper, @@ -114,6 +116,9 @@ self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths) self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory) self.__populate_unstructured_path_rewrites(param_dict) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized self.__populate_non_job_params(param_dict) # Return the dictionary of parameters @@ -334,6 +339,24 @@ #the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.tool or not self.tool.options or self.tool.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) + def build( self ): """ Build runtime description of job to execute, evaluate command and diff -r a74b0112995cd2ae396d0916a6e845b7709f9002 -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -2,6 +2,7 @@ from galaxy import exceptions from galaxy.util.none_like import NoneDataset from galaxy.util import odict +from galaxy.util.object_wrapper import wrap_with_safe_string from logging import getLogger log = getLogger( __name__ ) @@ -162,10 +163,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): @@ -190,9 +194,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry=datatypes_registry, ext=ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.datatypes_registry = datatypes_registry self.false_path = getattr( dataset_path, "false_path", None ) @@ -210,7 +218,7 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': @@ -230,7 +238,7 @@ # object store to find the static location of this # directory. try: - return self.dataset.extra_files_path + return self.unsanitized.extra_files_path except exceptions.ObjectNotFound: # NestedObjectstore raises an error here # instead of just returning a non-existent diff -r a74b0112995cd2ae396d0916a6e845b7709f9002 -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -360,46 +360,57 @@ '#': '__pd__'} -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) - -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: raise Exception('Unknown parameter type (%s)' % ( type( value ) )) diff -r a74b0112995cd2ae396d0916a6e845b7709f9002 -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 lib/galaxy/util/dbkeys.py --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -4,6 +4,7 @@ #dbkeys read from disk using builds.txt from galaxy.util import read_dbnames from galaxy.util.json import loads +from galaxy.util.object_wrapper import sanitize_lists_to_string import os.path @@ -84,6 +85,7 @@ # use configured server len path if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % dbkey ) + # Since we are using an unverified dbkey, we will sanitize the dbkey before use + chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string( dbkey ) ) chrom_info = os.path.abspath( chrom_info ) return ( chrom_info, db_dataset ) diff -r a74b0112995cd2ae396d0916a6e845b7709f9002 -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + diff -r a74b0112995cd2ae396d0916a6e845b7709f9002 -r 659fc6822f0afcdf77ba5f8857ba58700ce06b16 lib/galaxy/webapps/galaxy/controllers/tool_runner.py --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -80,14 +80,28 @@ message=message, status=status, redirect=redirect ) ) - params = galaxy.util.Params( kwd, sanitize = False ) #Sanitize parameters when substituting into command line via input wrappers - #do param translation here, used by datasource tools - if tool.input_translator: - tool.input_translator.translate( params ) + + def _validated_params_for( kwd ): + params = galaxy.util.Params( kwd, sanitize=False ) # Sanitize parameters when substituting into command line via input wrappers + #do param translation here, used by datasource tools + if tool.input_translator: + tool.input_translator.translate( params ) + return params + + params = _validated_params_for( kwd ) # We may be visiting Galaxy for the first time ( e.g., sending data from UCSC ), # so make sure to create a new history if we've never had one before. history = tool.get_default_history_by_trans( trans, create=True ) - template, vars = tool.handle_input( trans, params.__dict__ ) + try: + template, vars = tool.handle_input( trans, params.__dict__ ) + except KeyError: + # This error indicates (or at least can indicate) there was a + # problem with the stored tool_state - it is incompatible with + # this variant of the tool - possibly because the tool changed + # or because the tool version changed. + del kwd[ "tool_state" ] + params = _validated_params_for( kwd ) + template, vars = tool.handle_input( trans, params.__dict__ ) if len( params ) > 0: trans.log_event( "Tool params: %s" % ( str( params ) ), tool_id=tool_id ) add_frame = AddFrameData() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] b1ae13: Update Main
by GitHub 13 Jan '15

13 Jan '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: b1ae131ed466098f65a7b8a9346ce16d1d9faa69 https://github.com/galaxyproject/usegalaxy-playbook/commit/b1ae131ed466098f… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2015-01-13 (Tue, 13 Jan 2015) Changed paths: M production/group_vars/all.yml Log Message: ----------- Update Main

1 0

[galaxyproject/usegalaxy-playbook] ca6319: Update Test, commit mutable configs.
by GitHub 13 Jan '15

13 Jan '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: ca6319537b9bb471315ece8b2a4cd0ad07701e9d https://github.com/galaxyproject/usegalaxy-playbook/commit/ca6319537b9bb471… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2015-01-13 (Tue, 13 Jan 2015) Changed paths: M files/galaxy/test.galaxyproject.org/var/integrated_tool_panel.xml M files/galaxy/test.galaxyproject.org/var/migrated_tools_conf.xml M files/galaxy/test.galaxyproject.org/var/shed_data_manager_conf.xml M files/galaxy/test.galaxyproject.org/var/shed_tool_conf.xml M files/galaxy/test.galaxyproject.org/var/shed_tool_data_table_conf.xml M stage/group_vars/all.yml Log Message: ----------- Update Test, commit mutable configs.

1 0

commit/galaxy-central: 37 new changesets
by commits-noreply＠bitbucket.org 13 Jan '15

13 Jan '15

37 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9c323aad4ffd/ Changeset: 9c323aad4ffd Branch: stable User: dan Date: 2015-01-13 15:25:57+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 5 files diff -r 6b0bd93038a843b1585155f0d63f0eea2459c70b -r 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -2,6 +2,7 @@ from os.path import abspath from galaxy.util import string_as_bool, stringify_dictionary_keys, listify +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.web import form_builder import galaxy.model @@ -176,7 +177,10 @@ def to_string( self, value ): return str( value ) - + + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -394,6 +398,10 @@ def to_string( self, value ): return simplejson.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return simplejson.dumps( value ) + class PythonObjectParameter( MetadataParameter ): def to_string( self, value ): @@ -417,7 +425,11 @@ if not value: return str( self.spec.no_value ) return value.file_name - + + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=str( value.id ) ) diff -r 6b0bd93038a843b1585155f0d63f0eea2459c70b -r 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -34,6 +34,8 @@ from galaxy.util import listify import galaxy.util.shed_util_common from galaxy.web import url_for +from galaxy.util.object_wrapper import wrap_with_safe_string +from galaxy import exceptions from galaxy.visualization.genome.visual_analytics import TracksterConfig @@ -2382,6 +2384,9 @@ # failed to pass; for tool writing convienence, provide a # NoneDataset param_dict[ out_name ] = NoneDataset( datatypes_registry = self.app.datatypes_registry, ext = output.format ) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper( self.app ) # More convienent access to app.config.new_file_path; we don't need to @@ -2399,6 +2404,23 @@ param_dict['__admin_users__'] = self.app.config.admin_users # Return the dictionary of parameters return param_dict + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.options or self.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) def build_param_file( self, param_dict, directory=None ): """ Build temporary file for file based parameter transfer if needed @@ -3078,10 +3100,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again - setattr( self, name, rval ) + setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): return self.metadata.__nonzero__() @@ -3102,9 +3127,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry = datatypes_registry, ext = ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = false_path @@ -3112,10 +3141,28 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path + #does not implement support for false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) def __nonzero__( self ): diff -r 6b0bd93038a843b1585155f0d63f0eea2459c70b -r 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -6,6 +6,7 @@ from galaxy.tools.parameters.grouping import * from galaxy.util.template import fill_template from galaxy.util.none_like import NoneDataset +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.web import url_for from galaxy.exceptions import ObjectInvalid import galaxy.tools @@ -214,7 +215,7 @@ if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey ) + chrom_info = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % ( sanitize_lists_to_string( input_dbkey ) ) ) incoming[ "chromInfo" ] = chrom_info inp_data.update( db_datasets ) diff -r 6b0bd93038a843b1585155f0d63f0eea2459c70b -r 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -187,37 +187,59 @@ '#' : '__pd__' } -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): - """Restricts the characters that are allowed in a text""" + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + """ + Restricts the characters that are allowed in text; accepts both strings + and lists of strings; non-string entities will be cast to strings. + """ + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) + +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + """Restricts the characters that are allowed in a string""" + out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - print value - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) - -valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) invalid_filenames = [ '', '.', '..' ] def sanitize_for_filename( text, default=None ): """ @@ -399,6 +421,28 @@ except: return default +def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Returns a bytestring version of 's', encoded as specified in 'encoding'. + + If strings_only is True, don't convert (some) non-string-like objects. + + Adapted from an older, simpler version of django.utils.encoding.smart_str. + """ + if strings_only and isinstance(s, (type(None), int)): + return s + if not isinstance(s, basestring): + try: + return str(s) + except UnicodeEncodeError: + return unicode(s).encode(encoding, errors) + elif isinstance(s, unicode): + return s.encode(encoding, errors) + elif s and encoding != 'utf-8': + return s.decode('utf-8', errors).encode(encoding, errors) + else: + return s + def object_to_string( obj ): return binascii.hexlify( obj ) diff -r 6b0bd93038a843b1585155f0d63f0eea2459c70b -r 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/f5d12a0ef90f/ Changeset: f5d12a0ef90f Branch: stable User: natefoo Date: 2015-01-13 15:26:03+00:00 Summary: Update tag latest_2013.01.13 for changeset 9c323aad4ffd Affected #: 1 file diff -r c25df9f3d31effd9c4abdbbffd40381f23fc6dc8 -r f5d12a0ef90f8244ade33db951e002518ed8a9fc .hgtags --- a/.hgtags +++ b/.hgtags @@ -10,7 +10,7 @@ 7e257c7b10badb65772b1528cb61d58175a42e47 release_2014.06.02 7a4d321c0e38fa263ea83d29a35a608c3181fcba latest_2014.06.02 9661b9d5d5b330483ae3ad2236410e0efaa7c500 latest_2014.04.14 -6b0bd93038a843b1585155f0d63f0eea2459c70b latest_2013.01.13 +9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f latest_2013.01.13 3e62060b14b9afc46f8e0ec02e1a4500d77db9e1 latest_2013.02.08 425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 latest_2013.04.01 9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 latest_2013.06.03 https://bitbucket.org/galaxy/galaxy-central/commits/cbcce50577bf/ Changeset: cbcce50577bf Branch: stable User: natefoo Date: 2015-01-13 15:26:04+00:00 Summary: Merge head created for security fix on latest_2013.01.13 Affected #: 5 files diff -r f5d12a0ef90f8244ade33db951e002518ed8a9fc -r cbcce50577bf98aa164355caf631568e7c4b56f4 lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/b986c184be88/ Changeset: b986c184be88 Branch: stable User: dan Date: 2015-01-13 15:26:12+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 5 files diff -r 3e62060b14b9afc46f8e0ec02e1a4500d77db9e1 -r b986c184be88947b5d1d90be7f36cfd2627dd938 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -2,6 +2,7 @@ from os.path import abspath from galaxy.util import string_as_bool, stringify_dictionary_keys, listify +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.web import form_builder import galaxy.model @@ -176,7 +177,10 @@ def to_string( self, value ): return str( value ) - + + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -394,6 +398,10 @@ def to_string( self, value ): return simplejson.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return simplejson.dumps( value ) + class PythonObjectParameter( MetadataParameter ): def to_string( self, value ): @@ -417,7 +425,11 @@ if not value: return str( self.spec.no_value ) return value.file_name - + + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=str( value.id ) ) diff -r 3e62060b14b9afc46f8e0ec02e1a4500d77db9e1 -r b986c184be88947b5d1d90be7f36cfd2627dd938 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -35,6 +35,8 @@ from galaxy.util import listify import galaxy.util.shed_util_common from galaxy.web import url_for +from galaxy.util.object_wrapper import wrap_with_safe_string +from galaxy import exceptions from paste import httpexceptions @@ -2384,6 +2386,9 @@ # failed to pass; for tool writing convienence, provide a # NoneDataset param_dict[ out_name ] = NoneDataset( datatypes_registry = self.app.datatypes_registry, ext = output.format ) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper( self.app ) # More convienent access to app.config.new_file_path; we don't need to @@ -2401,6 +2406,23 @@ param_dict['__admin_users__'] = self.app.config.admin_users # Return the dictionary of parameters return param_dict + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.options or self.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) def build_param_file( self, param_dict, directory=None ): """ Build temporary file for file based parameter transfer if needed @@ -3080,10 +3102,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again - setattr( self, name, rval ) + setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): return self.metadata.__nonzero__() @@ -3104,9 +3129,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry = datatypes_registry, ext = ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = false_path @@ -3114,10 +3143,28 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path + #does not implement support for false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) def __nonzero__( self ): diff -r 3e62060b14b9afc46f8e0ec02e1a4500d77db9e1 -r b986c184be88947b5d1d90be7f36cfd2627dd938 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -6,6 +6,7 @@ from galaxy.tools.parameters.grouping import * from galaxy.util.template import fill_template from galaxy.util.none_like import NoneDataset +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.web import url_for from galaxy.exceptions import ObjectInvalid import galaxy.tools @@ -214,7 +215,7 @@ if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey ) + chrom_info = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % ( sanitize_lists_to_string( input_dbkey ) ) ) incoming[ "chromInfo" ] = chrom_info inp_data.update( db_datasets ) diff -r 3e62060b14b9afc46f8e0ec02e1a4500d77db9e1 -r b986c184be88947b5d1d90be7f36cfd2627dd938 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -187,37 +187,59 @@ '#' : '__pd__' } -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): - """Restricts the characters that are allowed in a text""" + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + """ + Restricts the characters that are allowed in text; accepts both strings + and lists of strings; non-string entities will be cast to strings. + """ + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) + +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + """Restricts the characters that are allowed in a string""" + out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - print value - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) - -valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) invalid_filenames = [ '', '.', '..' ] def sanitize_for_filename( text, default=None ): """ @@ -399,6 +421,28 @@ except: return default +def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Returns a bytestring version of 's', encoded as specified in 'encoding'. + + If strings_only is True, don't convert (some) non-string-like objects. + + Adapted from an older, simpler version of django.utils.encoding.smart_str. + """ + if strings_only and isinstance(s, (type(None), int)): + return s + if not isinstance(s, basestring): + try: + return str(s) + except UnicodeEncodeError: + return unicode(s).encode(encoding, errors) + elif isinstance(s, unicode): + return s.encode(encoding, errors) + elif s and encoding != 'utf-8': + return s.decode('utf-8', errors).encode(encoding, errors) + else: + return s + def object_to_string( obj ): return binascii.hexlify( obj ) diff -r 3e62060b14b9afc46f8e0ec02e1a4500d77db9e1 -r b986c184be88947b5d1d90be7f36cfd2627dd938 lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/ccbe7f3db659/ Changeset: ccbe7f3db659 Branch: stable User: natefoo Date: 2015-01-13 15:26:18+00:00 Summary: Update tag latest_2013.02.08 for changeset b986c184be88 Affected #: 1 file diff -r cbcce50577bf98aa164355caf631568e7c4b56f4 -r ccbe7f3db659092c9d7df2845e4bab3f58a7a6b0 .hgtags --- a/.hgtags +++ b/.hgtags @@ -11,7 +11,7 @@ 7a4d321c0e38fa263ea83d29a35a608c3181fcba latest_2014.06.02 9661b9d5d5b330483ae3ad2236410e0efaa7c500 latest_2014.04.14 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f latest_2013.01.13 -3e62060b14b9afc46f8e0ec02e1a4500d77db9e1 latest_2013.02.08 +b986c184be88947b5d1d90be7f36cfd2627dd938 latest_2013.02.08 425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 latest_2013.04.01 9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 latest_2013.06.03 9ed84cd208e07e8985ec917cb025fcbbb09edcfb latest_2013.08.12 https://bitbucket.org/galaxy/galaxy-central/commits/7c4a6bf151a2/ Changeset: 7c4a6bf151a2 Branch: stable User: natefoo Date: 2015-01-13 15:26:20+00:00 Summary: Merge head created for security fix on latest_2013.02.08 Affected #: 2 files https://bitbucket.org/galaxy/galaxy-central/commits/dec9431d66b8/ Changeset: dec9431d66b8 Branch: stable User: dan Date: 2015-01-13 15:26:27+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 5 files diff -r 425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 -r dec9431d66b837a208e2f060d90afd913c721227 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -2,6 +2,7 @@ from os.path import abspath from galaxy.util import string_as_bool, stringify_dictionary_keys, listify +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.web import form_builder import galaxy.model @@ -176,7 +177,10 @@ def to_string( self, value ): return str( value ) - + + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -394,6 +398,10 @@ def to_string( self, value ): return simplejson.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return simplejson.dumps( value ) + class PythonObjectParameter( MetadataParameter ): def to_string( self, value ): @@ -417,7 +425,11 @@ if not value: return str( self.spec.no_value ) return value.file_name - + + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=str( value.id ) ) diff -r 425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 -r dec9431d66b837a208e2f060d90afd913c721227 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -57,6 +57,8 @@ from galaxy.web import url_for from galaxy.web.form_builder import SelectField from tool_shed.util import shed_util_common +from galaxy.util.object_wrapper import wrap_with_safe_string +from galaxy import exceptions log = logging.getLogger( __name__ ) @@ -2621,7 +2623,9 @@ return self.app.tool_data_tables[ table_name ].get_entry( query_attr, query_val, return_attr ) param_dict['__get_data_table_entry__'] = get_data_table_entry - + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper( self.app ) # More convienent access to app.config.new_file_path; we don't need to @@ -2639,6 +2643,23 @@ param_dict['__admin_users__'] = self.app.config.admin_users # Return the dictionary of parameters return param_dict + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.options or self.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) def build_param_file( self, param_dict, directory=None ): """ Build temporary file for file based parameter transfer if needed @@ -3421,10 +3442,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): return self.metadata.__nonzero__() @@ -3445,9 +3469,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry = datatypes_registry, ext = ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = false_path @@ -3455,10 +3483,28 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path + #does not implement support for false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) def __nonzero__( self ): diff -r 425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 -r dec9431d66b837a208e2f060d90afd913c721227 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -6,6 +6,7 @@ from galaxy.tools.parameters.grouping import * from galaxy.util.template import fill_template from galaxy.util.none_like import NoneDataset +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.web import url_for from galaxy.exceptions import ObjectInvalid import galaxy.tools @@ -214,7 +215,7 @@ if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey ) + chrom_info = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % ( sanitize_lists_to_string( input_dbkey ) ) ) incoming[ "chromInfo" ] = chrom_info inp_data.update( db_datasets ) diff -r 425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 -r dec9431d66b837a208e2f060d90afd913c721227 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -187,48 +187,60 @@ '#' : '__pd__' } -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - print value - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) -valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] def sanitize_for_filename( text, default=None ): """ @@ -425,6 +437,28 @@ except: return default +def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Returns a bytestring version of 's', encoded as specified in 'encoding'. + + If strings_only is True, don't convert (some) non-string-like objects. + + Adapted from an older, simpler version of django.utils.encoding.smart_str. + """ + if strings_only and isinstance(s, (type(None), int)): + return s + if not isinstance(s, basestring): + try: + return str(s) + except UnicodeEncodeError: + return unicode(s).encode(encoding, errors) + elif isinstance(s, unicode): + return s.encode(encoding, errors) + elif s and encoding != 'utf-8': + return s.decode('utf-8', errors).encode(encoding, errors) + else: + return s + def object_to_string( obj ): return binascii.hexlify( obj ) diff -r 425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 -r dec9431d66b837a208e2f060d90afd913c721227 lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/b979070e064a/ Changeset: b979070e064a Branch: stable User: natefoo Date: 2015-01-13 15:26:32+00:00 Summary: Update tag latest_2013.04.01 for changeset dec9431d66b8 Affected #: 1 file diff -r 7c4a6bf151a279cbc9b639e1804f0701d3b11ccf -r b979070e064ace7b70ece868977e72712f647a80 .hgtags --- a/.hgtags +++ b/.hgtags @@ -12,7 +12,7 @@ 9661b9d5d5b330483ae3ad2236410e0efaa7c500 latest_2014.04.14 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f latest_2013.01.13 b986c184be88947b5d1d90be7f36cfd2627dd938 latest_2013.02.08 -425009b3ff4d8b67d2812253b221f3c4f4a8d1e3 latest_2013.04.01 +dec9431d66b837a208e2f060d90afd913c721227 latest_2013.04.01 9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 latest_2013.06.03 9ed84cd208e07e8985ec917cb025fcbbb09edcfb latest_2013.08.12 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 latest_2013.11.04 https://bitbucket.org/galaxy/galaxy-central/commits/29a6bfeb4f46/ Changeset: 29a6bfeb4f46 Branch: stable User: natefoo Date: 2015-01-13 15:26:33+00:00 Summary: Merge head created for security fix on latest_2013.04.01 Affected #: 3 files https://bitbucket.org/galaxy/galaxy-central/commits/19e56e66b0b3/ Changeset: 19e56e66b0b3 Branch: stable User: dan Date: 2015-01-13 15:26:40+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 5 files diff -r 9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 -r 19e56e66b0b344c6e2afa4541f6988e4fdb9af29 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -20,6 +20,7 @@ import galaxy.model from galaxy.util import listify, stringify_dictionary_keys, string_as_bool +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.web import form_builder from sqlalchemy.orm import object_session @@ -187,7 +188,10 @@ def to_string( self, value ): return str( value ) - + + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -405,6 +409,10 @@ def to_string( self, value ): return simplejson.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return simplejson.dumps( value ) + class PythonObjectParameter( MetadataParameter ): def to_string( self, value ): @@ -428,7 +436,11 @@ if not value: return str( self.spec.no_value ) return value.file_name - + + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=str( value.id ) ) diff -r 9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 -r 19e56e66b0b344c6e2afa4541f6988e4fdb9af29 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -60,6 +60,8 @@ from galaxy.web.form_builder import SelectField from tool_shed.util import shed_util_common from .loader import load_tool, template_macro_params +from galaxy.util.object_wrapper import wrap_with_safe_string +from galaxy import exceptions log = logging.getLogger( __name__ ) @@ -2543,7 +2545,9 @@ return self.app.tool_data_tables[ table_name ].get_entry( query_attr, query_val, return_attr ) param_dict['__get_data_table_entry__'] = get_data_table_entry - + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper( self.app ) # More convienent access to app.config.new_file_path; we don't need to @@ -2562,6 +2566,23 @@ param_dict['__user__'] = RawObjectWrapper( param_dict.get( '__user__', None ) ) # Return the dictionary of parameters return param_dict + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.options or self.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) def build_param_file( self, param_dict, directory=None ): """ Build temporary file for file based parameter transfer if needed @@ -3357,10 +3378,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): return self.metadata.__nonzero__() @@ -3381,9 +3405,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry = datatypes_registry, ext = ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = false_path @@ -3391,10 +3419,28 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path + #does not implement support for false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) def __nonzero__( self ): diff -r 9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 -r 19e56e66b0b344c6e2afa4541f6988e4fdb9af29 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -6,6 +6,7 @@ from galaxy.tools.parameters.grouping import * from galaxy.util.template import fill_template from galaxy.util.none_like import NoneDataset +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.web import url_for from galaxy.exceptions import ObjectInvalid import galaxy.tools @@ -214,7 +215,7 @@ if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % input_dbkey ) + chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % ( sanitize_lists_to_string( input_dbkey ) ) ) incoming[ "chromInfo" ] = chrom_info inp_data.update( db_datasets ) diff -r 9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 -r 19e56e66b0b344c6e2afa4541f6988e4fdb9af29 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -249,48 +249,60 @@ '#' : '__pd__' } -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - print value - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) -valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] def sanitize_for_filename( text, default=None ): """ @@ -488,6 +500,28 @@ except: return default +def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Returns a bytestring version of 's', encoded as specified in 'encoding'. + + If strings_only is True, don't convert (some) non-string-like objects. + + Adapted from an older, simpler version of django.utils.encoding.smart_str. + """ + if strings_only and isinstance(s, (type(None), int)): + return s + if not isinstance(s, basestring): + try: + return str(s) + except UnicodeEncodeError: + return unicode(s).encode(encoding, errors) + elif isinstance(s, unicode): + return s.encode(encoding, errors) + elif s and encoding != 'utf-8': + return s.decode('utf-8', errors).encode(encoding, errors) + else: + return s + def object_to_string( obj ): return binascii.hexlify( obj ) diff -r 9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 -r 19e56e66b0b344c6e2afa4541f6988e4fdb9af29 lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/4b481505104e/ Changeset: 4b481505104e Branch: stable User: natefoo Date: 2015-01-13 15:26:45+00:00 Summary: Update tag latest_2013.06.03 for changeset 19e56e66b0b3 Affected #: 1 file diff -r 29a6bfeb4f468e76c84bea8bc5d445ebf08fdfee -r 4b481505104efe8a711532ad0878e7f5cb6786c0 .hgtags --- a/.hgtags +++ b/.hgtags @@ -13,7 +13,7 @@ 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f latest_2013.01.13 b986c184be88947b5d1d90be7f36cfd2627dd938 latest_2013.02.08 dec9431d66b837a208e2f060d90afd913c721227 latest_2013.04.01 -9713d86392ef985ffcdc39ff0c8ddf51a1f9ce47 latest_2013.06.03 +19e56e66b0b344c6e2afa4541f6988e4fdb9af29 latest_2013.06.03 9ed84cd208e07e8985ec917cb025fcbbb09edcfb latest_2013.08.12 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 latest_2013.11.04 2a756ca2cb1826db7796018e77d12e2dd7b67603 latest_2014.02.10 https://bitbucket.org/galaxy/galaxy-central/commits/91ca514f61ec/ Changeset: 91ca514f61ec Branch: stable User: natefoo Date: 2015-01-13 15:26:46+00:00 Summary: Merge head created for security fix on latest_2013.06.03 Affected #: 4 files https://bitbucket.org/galaxy/galaxy-central/commits/cee903b8b3ee/ Changeset: cee903b8b3ee Branch: stable User: dan Date: 2015-01-13 15:26:56+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 5 files diff -r 9ed84cd208e07e8985ec917cb025fcbbb09edcfb -r cee903b8b3eee9145627ee89742555dac581791e lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -20,6 +20,7 @@ import galaxy.model from galaxy.util import listify, stringify_dictionary_keys, string_as_bool +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.web import form_builder from sqlalchemy.orm import object_session @@ -187,7 +188,10 @@ def to_string( self, value ): return str( value ) - + + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -405,6 +409,10 @@ def to_string( self, value ): return simplejson.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return simplejson.dumps( value ) + class PythonObjectParameter( MetadataParameter ): def to_string( self, value ): @@ -428,7 +436,11 @@ if not value: return str( self.spec.no_value ) return value.file_name - + + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=str( value.id ) ) diff -r 9ed84cd208e07e8985ec917cb025fcbbb09edcfb -r cee903b8b3eee9145627ee89742555dac581791e lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -61,6 +61,8 @@ from galaxy.web.form_builder import SelectField from tool_shed.util import shed_util_common from .loader import load_tool, template_macro_params +from galaxy.util.object_wrapper import wrap_with_safe_string +from galaxy import exceptions log = logging.getLogger( __name__ ) @@ -2553,7 +2555,9 @@ return self.app.tool_data_tables[ table_name ].get_entry( query_attr, query_val, return_attr ) param_dict['__get_data_table_entry__'] = get_data_table_entry - + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper( self.app ) # More convienent access to app.config.new_file_path; we don't need to @@ -2572,6 +2576,23 @@ param_dict['__user__'] = RawObjectWrapper( param_dict.get( '__user__', None ) ) # Return the dictionary of parameters return param_dict + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.options or self.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) def build_param_file( self, param_dict, directory=None ): """ Build temporary file for file based parameter transfer if needed @@ -3369,10 +3390,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): return self.metadata.__nonzero__() @@ -3393,9 +3417,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry = datatypes_registry, ext = ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = false_path @@ -3403,10 +3431,28 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path + #does not implement support for false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) def __nonzero__( self ): diff -r 9ed84cd208e07e8985ec917cb025fcbbb09edcfb -r cee903b8b3eee9145627ee89742555dac581791e lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -10,6 +10,7 @@ from galaxy.util.none_like import NoneDataset from galaxy.util.odict import odict from galaxy.util.template import fill_template +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.web import url_for import logging @@ -217,7 +218,7 @@ if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % input_dbkey ) + chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % ( sanitize_lists_to_string( input_dbkey ) ) ) incoming[ "chromInfo" ] = chrom_info inp_data.update( db_datasets ) diff -r 9ed84cd208e07e8985ec917cb025fcbbb09edcfb -r cee903b8b3eee9145627ee89742555dac581791e lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -301,47 +301,60 @@ '#' : '__pd__' } -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) -valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] def sanitize_for_filename( text, default=None ): """ @@ -539,6 +552,28 @@ except: return default +def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Returns a bytestring version of 's', encoded as specified in 'encoding'. + + If strings_only is True, don't convert (some) non-string-like objects. + + Adapted from an older, simpler version of django.utils.encoding.smart_str. + """ + if strings_only and isinstance(s, (type(None), int)): + return s + if not isinstance(s, basestring): + try: + return str(s) + except UnicodeEncodeError: + return unicode(s).encode(encoding, errors) + elif isinstance(s, unicode): + return s.encode(encoding, errors) + elif s and encoding != 'utf-8': + return s.decode('utf-8', errors).encode(encoding, errors) + else: + return s + def object_to_string( obj ): return binascii.hexlify( obj ) diff -r 9ed84cd208e07e8985ec917cb025fcbbb09edcfb -r cee903b8b3eee9145627ee89742555dac581791e lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/f87251ad7e02/ Changeset: f87251ad7e02 Branch: stable User: natefoo Date: 2015-01-13 15:27:02+00:00 Summary: Update tag latest_2013.08.12 for changeset cee903b8b3ee Affected #: 1 file diff -r 91ca514f61ec74cecd7ef93a1f34b4fc618e0694 -r f87251ad7e0262908cb4f6e37e235610c16eaeb9 .hgtags --- a/.hgtags +++ b/.hgtags @@ -14,7 +14,7 @@ b986c184be88947b5d1d90be7f36cfd2627dd938 latest_2013.02.08 dec9431d66b837a208e2f060d90afd913c721227 latest_2013.04.01 19e56e66b0b344c6e2afa4541f6988e4fdb9af29 latest_2013.06.03 -9ed84cd208e07e8985ec917cb025fcbbb09edcfb latest_2013.08.12 +cee903b8b3eee9145627ee89742555dac581791e latest_2013.08.12 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 latest_2013.11.04 2a756ca2cb1826db7796018e77d12e2dd7b67603 latest_2014.02.10 ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 https://bitbucket.org/galaxy/galaxy-central/commits/0face4c7b1c9/ Changeset: 0face4c7b1c9 Branch: stable User: natefoo Date: 2015-01-13 15:27:03+00:00 Summary: Merge head created for security fix on latest_2013.08.12 Affected #: 3 files https://bitbucket.org/galaxy/galaxy-central/commits/7d5aa19a166c/ Changeset: 7d5aa19a166c Branch: stable User: dan Date: 2015-01-13 15:27:09+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 5 files diff -r 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 -r 7d5aa19a166cba9039e15f338a1e3fc924c43d3a lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -20,6 +20,7 @@ import galaxy.model from galaxy.util import listify, stringify_dictionary_keys, string_as_bool +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.web import form_builder from sqlalchemy.orm import object_session @@ -188,6 +189,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -405,6 +409,10 @@ def to_string( self, value ): return simplejson.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return simplejson.dumps( value ) + class PythonObjectParameter( MetadataParameter ): def to_string( self, value ): @@ -429,6 +437,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=str( value.id ) ) diff -r 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 -r 7d5aa19a166cba9039e15f338a1e3fc924c43d3a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -65,6 +65,8 @@ from galaxy.model.item_attrs import Dictifiable from tool_shed.util import shed_util_common from .loader import load_tool, template_macro_params +from galaxy.util.object_wrapper import wrap_with_safe_string +from galaxy import exceptions log = logging.getLogger( __name__ ) @@ -2560,7 +2562,9 @@ return self.app.tool_data_tables[ table_name ].get_entry( query_attr, query_val, return_attr ) param_dict['__get_data_table_entry__'] = get_data_table_entry - + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper( self.app ) # More convienent access to app.config.new_file_path; we don't need to @@ -2579,6 +2583,23 @@ param_dict['__user__'] = RawObjectWrapper( param_dict.get( '__user__', None ) ) # Return the dictionary of parameters return param_dict + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.options or self.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) def build_param_file( self, param_dict, directory=None ): """ Build temporary file for file based parameter transfer if needed @@ -3352,10 +3373,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): return self.metadata.__nonzero__() @@ -3376,9 +3400,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry = datatypes_registry, ext = ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = false_path @@ -3386,10 +3414,28 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path + #does not implement support for false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) def __nonzero__( self ): diff -r 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 -r 7d5aa19a166cba9039e15f338a1e3fc924c43d3a lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -10,6 +10,7 @@ from galaxy.util.none_like import NoneDataset from galaxy.util.odict import odict from galaxy.util.template import fill_template +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.web import url_for import logging @@ -224,7 +225,7 @@ if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % input_dbkey ) + chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % ( sanitize_lists_to_string( input_dbkey ) ) ) incoming[ "chromInfo" ] = chrom_info inp_data.update( db_datasets ) diff -r 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 -r 7d5aa19a166cba9039e15f338a1e3fc924c43d3a lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -309,47 +309,60 @@ '#' : '__pd__' } -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) -valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] def sanitize_for_filename( text, default=None ): """ @@ -574,6 +587,28 @@ return s +def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Returns a bytestring version of 's', encoded as specified in 'encoding'. + + If strings_only is True, don't convert (some) non-string-like objects. + + Adapted from an older, simpler version of django.utils.encoding.smart_str. + """ + if strings_only and isinstance(s, (type(None), int)): + return s + if not isinstance(s, basestring): + try: + return str(s) + except UnicodeEncodeError: + return unicode(s).encode(encoding, errors) + elif isinstance(s, unicode): + return s.encode(encoding, errors) + elif s and encoding != 'utf-8': + return s.decode('utf-8', errors).encode(encoding, errors) + else: + return s + def object_to_string( obj ): return binascii.hexlify( obj ) diff -r 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 -r 7d5aa19a166cba9039e15f338a1e3fc924c43d3a lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/48635612a356/ Changeset: 48635612a356 Branch: stable User: natefoo Date: 2015-01-13 15:27:13+00:00 Summary: Update tag latest_2013.11.04 for changeset 7d5aa19a166c Affected #: 1 file diff -r 0face4c7b1c9c1991fb7ab93abca5e0f47eece96 -r 48635612a3561e83aa77d604e44d0cea80969c2a .hgtags --- a/.hgtags +++ b/.hgtags @@ -15,7 +15,7 @@ dec9431d66b837a208e2f060d90afd913c721227 latest_2013.04.01 19e56e66b0b344c6e2afa4541f6988e4fdb9af29 latest_2013.06.03 cee903b8b3eee9145627ee89742555dac581791e latest_2013.08.12 -81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 latest_2013.11.04 +7d5aa19a166cba9039e15f338a1e3fc924c43d3a latest_2013.11.04 2a756ca2cb1826db7796018e77d12e2dd7b67603 latest_2014.02.10 ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 548ab24667d6206780237bd807f7d857a484c461 latest_2014.08.11 https://bitbucket.org/galaxy/galaxy-central/commits/5b0c93bcc791/ Changeset: 5b0c93bcc791 Branch: stable User: natefoo Date: 2015-01-13 15:27:15+00:00 Summary: Merge head created for security fix on latest_2013.11.04 Affected #: 4 files https://bitbucket.org/galaxy/galaxy-central/commits/0c000cc2f9c0/ Changeset: 0c000cc2f9c0 Branch: stable User: dan Date: 2015-01-13 15:27:19+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 6 files diff -r 2a756ca2cb1826db7796018e77d12e2dd7b67603 -r 0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -17,6 +17,7 @@ import galaxy.model from galaxy.util import listify, stringify_dictionary_keys, string_as_bool +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.util import in_directory from galaxy.web import form_builder @@ -209,6 +210,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -457,6 +461,10 @@ def to_string( self, value ): return json.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return json.dumps( value ) + class PythonObjectParameter( MetadataParameter ): @@ -487,6 +495,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context=None, other_values=None, **kwd ): context = context or {} other_values = other_values or {} diff -r 2a756ca2cb1826db7796018e77d12e2dd7b67603 -r 0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -9,6 +9,7 @@ from galaxy.util.none_like import NoneDataset from galaxy.util.odict import odict from galaxy.util.template import fill_template +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.web import url_for import logging @@ -176,7 +177,7 @@ if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % input_dbkey ) + chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % ( sanitize_lists_to_string( input_dbkey ) ) ) incoming[ "chromInfo" ] = os.path.abspath( chrom_info ) inp_data.update( db_datasets ) diff -r 2a756ca2cb1826db7796018e77d12e2dd7b67603 -r 0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -2,10 +2,12 @@ import tempfile from galaxy import model +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.bunch import Bunch from galaxy.util.none_like import NoneDataset from galaxy.util.template import fill_template from galaxy.tools.wrappers import ( + ToolParameterValueWrapper, DatasetFilenameWrapper, DatasetListWrapper, LibraryDatasetValueWrapper, @@ -109,6 +111,9 @@ self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths) self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory) self.__populate_unstructured_path_rewrites(param_dict) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized self.__populate_non_job_params(param_dict) # Return the dictionary of parameters @@ -309,6 +314,24 @@ #the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.tool or not self.tool.options or self.tool.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) + def build( self ): """ Build runtime description of job to execute, evaluate command and diff -r 2a756ca2cb1826db7796018e77d12e2dd7b67603 -r 0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -1,5 +1,7 @@ import pipes +from galaxy import exceptions from galaxy.util.none_like import NoneDataset +from galaxy.util.object_wrapper import wrap_with_safe_string class ToolParameterValueWrapper( object ): @@ -145,10 +147,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): @@ -173,9 +178,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry=datatypes_registry, ext=ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = getattr( dataset_path, "false_path", None ) self.false_extra_files_path = getattr( dataset_path, "false_extra_files_path", None ) @@ -184,13 +193,31 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path elif self.false_extra_files_path is not None and key == 'extra_files_path': + # Path to extra files was rewritten for this job. return self.false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) diff -r 2a756ca2cb1826db7796018e77d12e2dd7b67603 -r 0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -328,48 +328,62 @@ '\n' : '__cn__', '\r' : '__cr__', '\t' : '__tc__', - '#' : '__pd__' - } + '#': '__pd__'} -def restore_text(text): + +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] diff -r 2a756ca2cb1826db7796018e77d12e2dd7b67603 -r 0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/bf8b6aeb7fd7/ Changeset: bf8b6aeb7fd7 Branch: stable User: natefoo Date: 2015-01-13 15:27:22+00:00 Summary: Update tag latest_2014.02.10 for changeset 0c000cc2f9c0 Affected #: 1 file diff -r 5b0c93bcc791747b315038a66fdb785f9db4ea47 -r bf8b6aeb7fd7d6b7ae8f64b97f7d585df22817b6 .hgtags --- a/.hgtags +++ b/.hgtags @@ -16,7 +16,7 @@ 19e56e66b0b344c6e2afa4541f6988e4fdb9af29 latest_2013.06.03 cee903b8b3eee9145627ee89742555dac581791e latest_2013.08.12 7d5aa19a166cba9039e15f338a1e3fc924c43d3a latest_2013.11.04 -2a756ca2cb1826db7796018e77d12e2dd7b67603 latest_2014.02.10 +0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 latest_2014.02.10 ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 548ab24667d6206780237bd807f7d857a484c461 latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 https://bitbucket.org/galaxy/galaxy-central/commits/50867216bdc8/ Changeset: 50867216bdc8 Branch: stable User: natefoo Date: 2015-01-13 15:27:23+00:00 Summary: Merge head created for security fix on latest_2014.02.10 Affected #: 5 files https://bitbucket.org/galaxy/galaxy-central/commits/8f9dcac03369/ Changeset: 8f9dcac03369 Branch: stable User: dan Date: 2015-01-13 15:27:27+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 6 files diff -r 9661b9d5d5b330483ae3ad2236410e0efaa7c500 -r 8f9dcac033694e4cabcf5daae5cca1cfefbe967f lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -17,6 +17,7 @@ import galaxy.model from galaxy.util import listify, stringify_dictionary_keys, string_as_bool +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.util import in_directory from galaxy.web import form_builder @@ -209,6 +210,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -457,6 +461,10 @@ def to_string( self, value ): return json.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return json.dumps( value ) + class PythonObjectParameter( MetadataParameter ): @@ -487,6 +495,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context=None, other_values=None, **kwd ): context = context or {} other_values = other_values or {} diff -r 9661b9d5d5b330483ae3ad2236410e0efaa7c500 -r 8f9dcac033694e4cabcf5daae5cca1cfefbe967f lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -9,6 +9,7 @@ from galaxy.util.none_like import NoneDataset from galaxy.util.odict import odict from galaxy.util.template import fill_template +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.web import url_for import logging @@ -176,7 +177,7 @@ if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % input_dbkey ) + chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % ( sanitize_lists_to_string( input_dbkey ) ) ) incoming[ "chromInfo" ] = os.path.abspath( chrom_info ) inp_data.update( db_datasets ) diff -r 9661b9d5d5b330483ae3ad2236410e0efaa7c500 -r 8f9dcac033694e4cabcf5daae5cca1cfefbe967f lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -2,10 +2,12 @@ import tempfile from galaxy import model +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.bunch import Bunch from galaxy.util.none_like import NoneDataset from galaxy.util.template import fill_template from galaxy.tools.wrappers import ( + ToolParameterValueWrapper, DatasetFilenameWrapper, DatasetListWrapper, LibraryDatasetValueWrapper, @@ -109,6 +111,9 @@ self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths) self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory) self.__populate_unstructured_path_rewrites(param_dict) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized self.__populate_non_job_params(param_dict) # Return the dictionary of parameters @@ -309,6 +314,24 @@ #the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.tool or not self.tool.options or self.tool.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) + def build( self ): """ Build runtime description of job to execute, evaluate command and diff -r 9661b9d5d5b330483ae3ad2236410e0efaa7c500 -r 8f9dcac033694e4cabcf5daae5cca1cfefbe967f lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -1,5 +1,7 @@ import pipes +from galaxy import exceptions from galaxy.util.none_like import NoneDataset +from galaxy.util.object_wrapper import wrap_with_safe_string class ToolParameterValueWrapper( object ): @@ -145,10 +147,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): @@ -173,9 +178,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry=datatypes_registry, ext=ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = getattr( dataset_path, "false_path", None ) self.false_extra_files_path = getattr( dataset_path, "false_extra_files_path", None ) @@ -184,13 +193,31 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path elif self.false_extra_files_path is not None and key == 'extra_files_path': + # Path to extra files was rewritten for this job. return self.false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) diff -r 9661b9d5d5b330483ae3ad2236410e0efaa7c500 -r 8f9dcac033694e4cabcf5daae5cca1cfefbe967f lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -330,48 +330,62 @@ '\n' : '__cn__', '\r' : '__cr__', '\t' : '__tc__', - '#' : '__pd__' - } + '#': '__pd__'} -def restore_text(text): + +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] diff -r 9661b9d5d5b330483ae3ad2236410e0efaa7c500 -r 8f9dcac033694e4cabcf5daae5cca1cfefbe967f lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/1cc98e8f6782/ Changeset: 1cc98e8f6782 Branch: stable User: natefoo Date: 2015-01-13 15:27:31+00:00 Summary: Update tag latest_2014.04.14 for changeset 8f9dcac03369 Affected #: 1 file diff -r 50867216bdc89348afbce913b8371eb3463ce576 -r 1cc98e8f678283d05dbd5693bc0f48ce49c340bf .hgtags --- a/.hgtags +++ b/.hgtags @@ -9,7 +9,7 @@ 9e53251b0b7e93b9563008a2b112f2e815a04bbc release_2014.04.14 7e257c7b10badb65772b1528cb61d58175a42e47 release_2014.06.02 7a4d321c0e38fa263ea83d29a35a608c3181fcba latest_2014.06.02 -9661b9d5d5b330483ae3ad2236410e0efaa7c500 latest_2014.04.14 +8f9dcac033694e4cabcf5daae5cca1cfefbe967f latest_2014.04.14 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f latest_2013.01.13 b986c184be88947b5d1d90be7f36cfd2627dd938 latest_2013.02.08 dec9431d66b837a208e2f060d90afd913c721227 latest_2013.04.01 https://bitbucket.org/galaxy/galaxy-central/commits/c46ef9a4aa9c/ Changeset: c46ef9a4aa9c Branch: stable User: natefoo Date: 2015-01-13 15:27:32+00:00 Summary: Merge head created for security fix on latest_2014.04.14 Affected #: 1 file https://bitbucket.org/galaxy/galaxy-central/commits/4145417a6e1c/ Changeset: 4145417a6e1c Branch: stable User: dan Date: 2015-01-13 15:27:36+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 6 files diff -r 7a4d321c0e38fa263ea83d29a35a608c3181fcba -r 4145417a6e1c13f82a3de365aadef0fb3ed7ab14 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -17,6 +17,7 @@ import galaxy.model from galaxy.util import listify, stringify_dictionary_keys, string_as_bool +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.util import in_directory from galaxy.web import form_builder @@ -209,6 +210,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -457,6 +461,10 @@ def to_string( self, value ): return json.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return json.dumps( value ) + class PythonObjectParameter( MetadataParameter ): @@ -487,6 +495,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context=None, other_values=None, **kwd ): context = context or {} other_values = other_values or {} diff -r 7a4d321c0e38fa263ea83d29a35a608c3181fcba -r 4145417a6e1c13f82a3de365aadef0fb3ed7ab14 lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -2,10 +2,12 @@ import tempfile from galaxy import model +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.bunch import Bunch from galaxy.util.none_like import NoneDataset from galaxy.util.template import fill_template from galaxy.tools.wrappers import ( + ToolParameterValueWrapper, DatasetFilenameWrapper, DatasetListWrapper, DatasetCollectionWrapper, @@ -114,6 +116,9 @@ self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths) self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory) self.__populate_unstructured_path_rewrites(param_dict) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized self.__populate_non_job_params(param_dict) # Return the dictionary of parameters @@ -334,6 +339,24 @@ #the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.tool or not self.tool.options or self.tool.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) + def build( self ): """ Build runtime description of job to execute, evaluate command and diff -r 7a4d321c0e38fa263ea83d29a35a608c3181fcba -r 4145417a6e1c13f82a3de365aadef0fb3ed7ab14 lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -1,5 +1,7 @@ import pipes +from galaxy import exceptions from galaxy.util.none_like import NoneDataset +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util import odict from logging import getLogger @@ -149,10 +151,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): @@ -177,9 +182,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry=datatypes_registry, ext=ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = getattr( dataset_path, "false_path", None ) self.false_extra_files_path = getattr( dataset_path, "false_extra_files_path", None ) @@ -192,13 +201,31 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path elif self.false_extra_files_path is not None and key == 'extra_files_path': + # Path to extra files was rewritten for this job. return self.false_extra_files_path + elif key == 'extra_files_path': + try: + # Assume it is an output and that this wrapper + # will be set with correct "files_path" for this + # job. + return self.files_path + except AttributeError: + # Otherwise, we have an input - delegate to model and + # object store to find the static location of this + # directory. + try: + return self.unsanitized.extra_files_path + except exceptions.ObjectNotFound: + # NestedObjectstore raises an error here + # instead of just returning a non-existent + # path like DiskObjectStore. + raise else: return getattr( self.dataset, key ) diff -r 7a4d321c0e38fa263ea83d29a35a608c3181fcba -r 4145417a6e1c13f82a3de365aadef0fb3ed7ab14 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -330,48 +330,62 @@ '\n' : '__cn__', '\r' : '__cr__', '\t' : '__tc__', - '#' : '__pd__' - } + '#': '__pd__'} -def restore_text(text): + +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): + +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): + +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] diff -r 7a4d321c0e38fa263ea83d29a35a608c3181fcba -r 4145417a6e1c13f82a3de365aadef0fb3ed7ab14 lib/galaxy/util/dbkeys.py --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -4,6 +4,7 @@ #dbkeys read from disk using builds.txt from galaxy.util import dbnames from galaxy.util.json import from_json_string +from galaxy.util.object_wrapper import sanitize_lists_to_string import os.path @@ -84,6 +85,7 @@ # use configured server len path if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % dbkey ) + # Since we are using an unverified dbkey, we will sanitize the dbkey before use + chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string( dbkey ) ) chrom_info = os.path.abspath( chrom_info ) return ( chrom_info, db_dataset ) diff -r 7a4d321c0e38fa263ea83d29a35a608c3181fcba -r 4145417a6e1c13f82a3de365aadef0fb3ed7ab14 lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/5d390ca2627a/ Changeset: 5d390ca2627a Branch: stable User: natefoo Date: 2015-01-13 15:27:39+00:00 Summary: Update tag latest_2014.06.02 for changeset 4145417a6e1c Affected #: 1 file diff -r c46ef9a4aa9cb7eca5951d82bf9aeba408a48496 -r 5d390ca2627a4e9b7ab004da072f21b15dc25501 .hgtags --- a/.hgtags +++ b/.hgtags @@ -8,7 +8,7 @@ 5e605ed6069fe4c5ca9875e95e91b2713499e8ca release_2014.02.10 9e53251b0b7e93b9563008a2b112f2e815a04bbc release_2014.04.14 7e257c7b10badb65772b1528cb61d58175a42e47 release_2014.06.02 -7a4d321c0e38fa263ea83d29a35a608c3181fcba latest_2014.06.02 +4145417a6e1c13f82a3de365aadef0fb3ed7ab14 latest_2014.06.02 8f9dcac033694e4cabcf5daae5cca1cfefbe967f latest_2014.04.14 9c323aad4ffdd65a3deb06a4a36f6b2c5115a60f latest_2013.01.13 b986c184be88947b5d1d90be7f36cfd2627dd938 latest_2013.02.08 https://bitbucket.org/galaxy/galaxy-central/commits/ffacc6553f1a/ Changeset: ffacc6553f1a Branch: stable User: natefoo Date: 2015-01-13 15:27:40+00:00 Summary: Merge head created for security fix on latest_2014.06.02 Affected #: 4 files https://bitbucket.org/galaxy/galaxy-central/commits/8150024c0e6f/ Changeset: 8150024c0e6f Branch: stable User: dan Date: 2015-01-13 15:27:43+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 6 files diff -r 548ab24667d6206780237bd807f7d857a484c461 -r 8150024c0e6fc5aef3033cf8aaa574896f6b5d0d lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -17,6 +17,7 @@ import galaxy.model from galaxy.util import listify, stringify_dictionary_keys, string_as_bool +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util.odict import odict from galaxy.util import in_directory from galaxy.web import form_builder @@ -227,6 +228,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -475,6 +479,10 @@ def to_string( self, value ): return json.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return json.dumps( value ) + class PythonObjectParameter( MetadataParameter ): @@ -505,6 +513,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context=None, other_values=None, **kwd ): context = context or {} other_values = other_values or {} diff -r 548ab24667d6206780237bd807f7d857a484c461 -r 8150024c0e6fc5aef3033cf8aaa574896f6b5d0d lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -2,10 +2,12 @@ import tempfile from galaxy import model +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.bunch import Bunch from galaxy.util.none_like import NoneDataset from galaxy.util.template import fill_template from galaxy.tools.wrappers import ( + ToolParameterValueWrapper, DatasetFilenameWrapper, DatasetListWrapper, DatasetCollectionWrapper, @@ -114,6 +116,9 @@ self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths) self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory) self.__populate_unstructured_path_rewrites(param_dict) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized self.__populate_non_job_params(param_dict) # Return the dictionary of parameters @@ -334,6 +339,24 @@ #the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.tool or not self.tool.options or self.tool.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) + def build( self ): """ Build runtime description of job to execute, evaluate command and diff -r 548ab24667d6206780237bd807f7d857a484c461 -r 8150024c0e6fc5aef3033cf8aaa574896f6b5d0d lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -1,6 +1,7 @@ import pipes from galaxy import exceptions from galaxy.util.none_like import NoneDataset +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util import odict from logging import getLogger @@ -162,10 +163,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): @@ -190,9 +194,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry=datatypes_registry, ext=ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = getattr( dataset_path, "false_path", None ) self.false_extra_files_path = getattr( dataset_path, "false_extra_files_path", None ) @@ -205,7 +213,7 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': @@ -225,7 +233,7 @@ # object store to find the static location of this # directory. try: - return self.dataset.extra_files_path + return self.unsanitized.extra_files_path except exceptions.ObjectNotFound: # NestedObjectstore raises an error here # instead of just returning a non-existent diff -r 548ab24667d6206780237bd807f7d857a484c461 -r 8150024c0e6fc5aef3033cf8aaa574896f6b5d0d lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -345,46 +345,57 @@ '#': '__pd__'} -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) - -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: raise Exception('Unknown parameter type (%s)' % ( type( value ) )) diff -r 548ab24667d6206780237bd807f7d857a484c461 -r 8150024c0e6fc5aef3033cf8aaa574896f6b5d0d lib/galaxy/util/dbkeys.py --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -4,6 +4,7 @@ #dbkeys read from disk using builds.txt from galaxy.util import dbnames from galaxy.util.json import from_json_string +from galaxy.util.object_wrapper import sanitize_lists_to_string import os.path @@ -84,6 +85,7 @@ # use configured server len path if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % dbkey ) + # Since we are using an unverified dbkey, we will sanitize the dbkey before use + chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string( dbkey ) ) chrom_info = os.path.abspath( chrom_info ) return ( chrom_info, db_dataset ) diff -r 548ab24667d6206780237bd807f7d857a484c461 -r 8150024c0e6fc5aef3033cf8aaa574896f6b5d0d lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/c776fba69528/ Changeset: c776fba69528 Branch: stable User: natefoo Date: 2015-01-13 15:27:46+00:00 Summary: Update tag latest_2014.08.11 for changeset 8150024c0e6f Affected #: 1 file diff -r ffacc6553f1ad02470d92e058c15848251e12453 -r c776fba69528715271b416a97928246888180d80 .hgtags --- a/.hgtags +++ b/.hgtags @@ -18,6 +18,6 @@ 7d5aa19a166cba9039e15f338a1e3fc924c43d3a latest_2013.11.04 0c000cc2f9c05bf4c1c2bc3a10215014fd64e696 latest_2014.02.10 ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 -548ab24667d6206780237bd807f7d857a484c461 latest_2014.08.11 +8150024c0e6fc5aef3033cf8aaa574896f6b5d0d latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 ff6e36d7a2388214fe7636c43d30838c07246907 latest_2014.10.06 https://bitbucket.org/galaxy/galaxy-central/commits/37201f6fe299/ Changeset: 37201f6fe299 Branch: stable User: natefoo Date: 2015-01-13 15:27:47+00:00 Summary: Merge head created for security fix on latest_2014.08.11 Affected #: 3 files https://bitbucket.org/galaxy/galaxy-central/commits/c437b28348a9/ Changeset: c437b28348a9 Branch: stable User: dan Date: 2015-01-13 15:27:49+00:00 Summary: Fix a critical security vulnerability where unsanitized user-modifiable values could be included in a command line template. Affected #: 6 files diff -r ff6e36d7a2388214fe7636c43d30838c07246907 -r c437b28348a9345db8433e5b4f0e05ec8fb6c38a lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -20,6 +20,7 @@ import galaxy.model from galaxy.util import listify +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util import stringify_dictionary_keys from galaxy.util import string_as_bool from galaxy.util import in_directory @@ -232,6 +233,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -480,6 +484,10 @@ def to_string( self, value ): return json.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return json.safe_dumps( value ) + class PythonObjectParameter( MetadataParameter ): @@ -510,6 +518,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context=None, other_values=None, **kwd ): context = context or {} other_values = other_values or {} diff -r ff6e36d7a2388214fe7636c43d30838c07246907 -r c437b28348a9345db8433e5b4f0e05ec8fb6c38a lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -2,10 +2,12 @@ import tempfile from galaxy import model +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.bunch import Bunch from galaxy.util.none_like import NoneDataset from galaxy.util.template import fill_template from galaxy.tools.wrappers import ( + ToolParameterValueWrapper, DatasetFilenameWrapper, DatasetListWrapper, DatasetCollectionWrapper, @@ -114,6 +116,9 @@ self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths) self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory) self.__populate_unstructured_path_rewrites(param_dict) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized self.__populate_non_job_params(param_dict) # Return the dictionary of parameters @@ -334,6 +339,24 @@ #the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.tool or not self.tool.options or self.tool.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) + def build( self ): """ Build runtime description of job to execute, evaluate command and diff -r ff6e36d7a2388214fe7636c43d30838c07246907 -r c437b28348a9345db8433e5b4f0e05ec8fb6c38a lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -2,6 +2,7 @@ from galaxy import exceptions from galaxy.util.none_like import NoneDataset from galaxy.util import odict +from galaxy.util.object_wrapper import wrap_with_safe_string from logging import getLogger log = getLogger( __name__ ) @@ -162,10 +163,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): @@ -190,9 +194,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry=datatypes_registry, ext=ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.datatypes_registry = datatypes_registry self.false_path = getattr( dataset_path, "false_path", None ) @@ -210,7 +218,7 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': @@ -230,7 +238,7 @@ # object store to find the static location of this # directory. try: - return self.dataset.extra_files_path + return self.unsanitized.extra_files_path except exceptions.ObjectNotFound: # NestedObjectstore raises an error here # instead of just returning a non-existent diff -r ff6e36d7a2388214fe7636c43d30838c07246907 -r c437b28348a9345db8433e5b4f0e05ec8fb6c38a lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -360,46 +360,57 @@ '#': '__pd__'} -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) - -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: raise Exception('Unknown parameter type (%s)' % ( type( value ) )) diff -r ff6e36d7a2388214fe7636c43d30838c07246907 -r c437b28348a9345db8433e5b4f0e05ec8fb6c38a lib/galaxy/util/dbkeys.py --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -4,6 +4,7 @@ #dbkeys read from disk using builds.txt from galaxy.util import read_dbnames from galaxy.util.json import loads +from galaxy.util.object_wrapper import sanitize_lists_to_string import os.path @@ -84,6 +85,7 @@ # use configured server len path if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % dbkey ) + # Since we are using an unverified dbkey, we will sanitize the dbkey before use + chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string( dbkey ) ) chrom_info = os.path.abspath( chrom_info ) return ( chrom_info, db_dataset ) diff -r ff6e36d7a2388214fe7636c43d30838c07246907 -r c437b28348a9345db8433e5b4f0e05ec8fb6c38a lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/593bd69b3d5b/ Changeset: 593bd69b3d5b Branch: stable User: natefoo Date: 2015-01-13 15:27:50+00:00 Summary: Update tag latest_2014.10.06 for changeset c437b28348a9 Affected #: 1 file diff -r 37201f6fe299a099fad0b5f54ff0ba72a02185cb -r 593bd69b3d5b937c2eb0ae5133a8f989b3131504 .hgtags --- a/.hgtags +++ b/.hgtags @@ -20,4 +20,4 @@ ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 8150024c0e6fc5aef3033cf8aaa574896f6b5d0d latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 -ff6e36d7a2388214fe7636c43d30838c07246907 latest_2014.10.06 +c437b28348a9345db8433e5b4f0e05ec8fb6c38a latest_2014.10.06 https://bitbucket.org/galaxy/galaxy-central/commits/605de23ca239/ Changeset: 605de23ca239 Branch: stable User: natefoo Date: 2015-01-13 15:27:51+00:00 Summary: Merge head created for security fix on latest_2014.10.06 Affected #: 5 files diff -r 593bd69b3d5b937c2eb0ae5133a8f989b3131504 -r 605de23ca239879a34426885feddd2d3a459ca26 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -20,6 +20,7 @@ import galaxy.model from galaxy.util import listify +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util import stringify_dictionary_keys from galaxy.util import string_as_bool from galaxy.util import in_directory @@ -232,6 +233,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -480,6 +484,10 @@ def to_string( self, value ): return json.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return json.safe_dumps( value ) + class PythonObjectParameter( MetadataParameter ): @@ -510,6 +518,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context=None, other_values=None, **kwd ): context = context or {} other_values = other_values or {} diff -r 593bd69b3d5b937c2eb0ae5133a8f989b3131504 -r 605de23ca239879a34426885feddd2d3a459ca26 lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -2,10 +2,12 @@ import tempfile from galaxy import model +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.bunch import Bunch from galaxy.util.none_like import NoneDataset from galaxy.util.template import fill_template from galaxy.tools.wrappers import ( + ToolParameterValueWrapper, DatasetFilenameWrapper, DatasetListWrapper, DatasetCollectionWrapper, @@ -114,6 +116,9 @@ self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths) self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory) self.__populate_unstructured_path_rewrites(param_dict) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized self.__populate_non_job_params(param_dict) # Return the dictionary of parameters @@ -334,6 +339,24 @@ #the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.tool or not self.tool.options or self.tool.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) + def build( self ): """ Build runtime description of job to execute, evaluate command and diff -r 593bd69b3d5b937c2eb0ae5133a8f989b3131504 -r 605de23ca239879a34426885feddd2d3a459ca26 lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -2,6 +2,7 @@ from galaxy import exceptions from galaxy.util.none_like import NoneDataset from galaxy.util import odict +from galaxy.util.object_wrapper import wrap_with_safe_string from logging import getLogger log = getLogger( __name__ ) @@ -162,10 +163,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): @@ -190,9 +194,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry=datatypes_registry, ext=ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.datatypes_registry = datatypes_registry self.false_path = getattr( dataset_path, "false_path", None ) @@ -210,7 +218,7 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': @@ -230,7 +238,7 @@ # object store to find the static location of this # directory. try: - return self.dataset.extra_files_path + return self.unsanitized.extra_files_path except exceptions.ObjectNotFound: # NestedObjectstore raises an error here # instead of just returning a non-existent diff -r 593bd69b3d5b937c2eb0ae5133a8f989b3131504 -r 605de23ca239879a34426885feddd2d3a459ca26 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -360,46 +360,57 @@ '#': '__pd__'} -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) - -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: raise Exception('Unknown parameter type (%s)' % ( type( value ) )) diff -r 593bd69b3d5b937c2eb0ae5133a8f989b3131504 -r 605de23ca239879a34426885feddd2d3a459ca26 lib/galaxy/util/dbkeys.py --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -4,6 +4,7 @@ #dbkeys read from disk using builds.txt from galaxy.util import read_dbnames from galaxy.util.json import loads +from galaxy.util.object_wrapper import sanitize_lists_to_string import os.path @@ -84,6 +85,7 @@ # use configured server len path if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % dbkey ) + # Since we are using an unverified dbkey, we will sanitize the dbkey before use + chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string( dbkey ) ) chrom_info = os.path.abspath( chrom_info ) return ( chrom_info, db_dataset ) https://bitbucket.org/galaxy/galaxy-central/commits/8da5fd3df6e7/ Changeset: 8da5fd3df6e7 Branch: next-stable User: natefoo Date: 2015-01-13 15:33:41+00:00 Summary: Close next-stable branch for release_2015.01.13 Affected #: 6 files diff -r 7fe5cbb6b66e0debec7775a88a8c553384784a00 -r 8da5fd3df6e7e710bd9c9e5acc3baa98a665e247 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -20,6 +20,7 @@ import galaxy.model from galaxy.util import listify +from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util import stringify_dictionary_keys from galaxy.util import string_as_bool from galaxy.util import in_directory @@ -232,6 +233,9 @@ def to_string( self, value ): return str( value ) + def to_safe_string( self, value ): + return sanitize_lists_to_string( self.to_string( value ) ) + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) @@ -480,6 +484,10 @@ def to_string( self, value ): return json.dumps( value ) + def to_safe_string( self, value ): + # We do not sanitize json dicts + return json.safe_dumps( value ) + class PythonObjectParameter( MetadataParameter ): @@ -510,6 +518,10 @@ return str( self.spec.no_value ) return value.file_name + def to_safe_string( self, value ): + # We do not sanitize file names + return self.to_string( value ) + def get_html_field( self, value=None, context=None, other_values=None, **kwd ): context = context or {} other_values = other_values or {} diff -r 7fe5cbb6b66e0debec7775a88a8c553384784a00 -r 8da5fd3df6e7e710bd9c9e5acc3baa98a665e247 lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -2,10 +2,12 @@ import tempfile from galaxy import model +from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.bunch import Bunch from galaxy.util.none_like import NoneDataset from galaxy.util.template import fill_template from galaxy.tools.wrappers import ( + ToolParameterValueWrapper, DatasetFilenameWrapper, DatasetListWrapper, DatasetCollectionWrapper, @@ -114,6 +116,9 @@ self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths) self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory) self.__populate_unstructured_path_rewrites(param_dict) + # Call param dict sanitizer, before non-job params are added, as we don't want to sanitize filenames. + self.__sanitize_param_dict( param_dict ) + # Parameters added after this line are not sanitized self.__populate_non_job_params(param_dict) # Return the dictionary of parameters @@ -334,6 +339,24 @@ #the paths rewritten. self.__walk_inputs( self.tool.inputs, param_dict, rewrite_unstructured_paths ) + def __sanitize_param_dict( self, param_dict ): + """ + Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers, + which already have their own specific sanitization rules and also exclude special-cased named values. + We will only examine the first level for values to skip; the wrapping function will recurse as necessary. + + Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place. + """ + # chromInfo is a filename, do not sanitize it. + skip = [ 'chromInfo' ] + if not self.tool or not self.tool.options or self.tool.options.sanitize: + for key, value in param_dict.items(): + if key not in skip: + # Remove key so that new wrapped object will occupy key slot + del param_dict[key] + # And replace with new wrapped key + param_dict[ wrap_with_safe_string( key, no_wrap_classes=ToolParameterValueWrapper ) ] = wrap_with_safe_string( value, no_wrap_classes=ToolParameterValueWrapper ) + def build( self ): """ Build runtime description of job to execute, evaluate command and diff -r 7fe5cbb6b66e0debec7775a88a8c553384784a00 -r 8da5fd3df6e7e710bd9c9e5acc3baa98a665e247 lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -2,6 +2,7 @@ from galaxy import exceptions from galaxy.util.none_like import NoneDataset from galaxy.util import odict +from galaxy.util.object_wrapper import wrap_with_safe_string from logging import getLogger log = getLogger( __name__ ) @@ -162,10 +163,13 @@ if name in self.metadata.spec: if rval is None: rval = self.metadata.spec[name].no_value - rval = self.metadata.spec[name].param.to_string( rval ) + rval = self.metadata.spec[ name ].param.to_safe_string( rval ) # Store this value, so we don't need to recalculate if needed # again setattr( self, name, rval ) + else: + #escape string value of non-defined metadata value + rval = wrap_with_safe_string( rval ) return rval def __nonzero__( self ): @@ -190,9 +194,13 @@ ext = tool.inputs[name].extensions[0] except: ext = 'data' - self.dataset = NoneDataset( datatypes_registry=datatypes_registry, ext=ext ) + self.dataset = wrap_with_safe_string( NoneDataset( datatypes_registry=datatypes_registry, ext=ext ), no_wrap_classes=ToolParameterValueWrapper ) else: - self.dataset = dataset + # Tool wrappers should not normally be accessing .dataset directly, + # so we will wrap it and keep the original around for file paths + # Should we name this .value to maintain consistency with most other ToolParameterValueWrapper? + self.unsanitized = dataset + self.dataset = wrap_with_safe_string( dataset, no_wrap_classes=ToolParameterValueWrapper ) self.metadata = self.MetadataWrapper( dataset.metadata ) self.datatypes_registry = datatypes_registry self.false_path = getattr( dataset_path, "false_path", None ) @@ -210,7 +218,7 @@ if self.false_path is not None: return self.false_path else: - return self.dataset.file_name + return self.unsanitized.file_name def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': @@ -230,7 +238,7 @@ # object store to find the static location of this # directory. try: - return self.dataset.extra_files_path + return self.unsanitized.extra_files_path except exceptions.ObjectNotFound: # NestedObjectstore raises an error here # instead of just returning a non-existent diff -r 7fe5cbb6b66e0debec7775a88a8c553384784a00 -r 8da5fd3df6e7e710bd9c9e5acc3baa98a665e247 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -360,46 +360,57 @@ '#': '__pd__'} -def restore_text(text): +def restore_text( text, character_map=mapped_chars ): """Restores sanitized text""" if not text: return text - for key, value in mapped_chars.items(): + for key, value in character_map.items(): text = text.replace(value, key) return text -def sanitize_text(text): +def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """ Restricts the characters that are allowed in text; accepts both strings - and lists of strings. + and lists of strings; non-string entities will be cast to strings. """ - if isinstance( text, basestring ): - return _sanitize_text_helper(text) - elif isinstance( text, list ): - return [ _sanitize_text_helper(t) for t in text ] + if isinstance( text, list ): + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + if not isinstance( text, basestring ): + text = smart_str( text ) + return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) - -def _sanitize_text_helper(text): +def _sanitize_text_helper( text, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Restricts the characters that are allowed in a string""" out = [] for c in text: - if c in valid_chars: + if c in valid_characters: out.append(c) - elif c in mapped_chars: - out.append(mapped_chars[c]) + elif c in character_map: + out.append( character_map[c] ) else: - out.append('X') # makes debugging easier + out.append( invalid_character ) # makes debugging easier return ''.join(out) -def sanitize_param(value): +def sanitize_lists_to_string( values, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): + if isinstance( values, list ): + rval = [] + for value in values: + rval.append( sanitize_lists_to_string( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) ) + values = ",".join( rval ) + else: + values = sanitize_text( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + return values + + +def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_chars, invalid_character='X' ): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): - return sanitize_text(value) + return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map(sanitize_text, value) + return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: raise Exception('Unknown parameter type (%s)' % ( type( value ) )) diff -r 7fe5cbb6b66e0debec7775a88a8c553384784a00 -r 8da5fd3df6e7e710bd9c9e5acc3baa98a665e247 lib/galaxy/util/dbkeys.py --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -4,6 +4,7 @@ #dbkeys read from disk using builds.txt from galaxy.util import read_dbnames from galaxy.util.json import loads +from galaxy.util.object_wrapper import sanitize_lists_to_string import os.path @@ -84,6 +85,7 @@ # use configured server len path if not chrom_info: # Default to built-in build. - chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % dbkey ) + # Since we are using an unverified dbkey, we will sanitize the dbkey before use + chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string( dbkey ) ) chrom_info = os.path.abspath( chrom_info ) return ( chrom_info, db_dataset ) diff -r 7fe5cbb6b66e0debec7775a88a8c553384784a00 -r 8da5fd3df6e7e710bd9c9e5acc3baa98a665e247 lib/galaxy/util/object_wrapper.py --- /dev/null +++ b/lib/galaxy/util/object_wrapper.py @@ -0,0 +1,436 @@ +""" +Classes for wrapping Objects and Sanitizing string output. +""" + +import inspect +import copy_reg +import logging +import string +from numbers import Number +from types import ( NoneType, NotImplementedType, EllipsisType, FunctionType, MethodType, GeneratorType, CodeType, + BuiltinFunctionType, BuiltinMethodType, ModuleType, XRangeType, SliceType, TracebackType, FrameType, + BufferType, DictProxyType, GetSetDescriptorType, MemberDescriptorType ) +from UserDict import UserDict + +from galaxy.util import sanitize_lists_to_string as _sanitize_lists_to_string + +log = logging.getLogger( __name__ ) + +# Define different behaviors for different types, see also: https://docs.python.org/2/library/types.html + +# Known Callable types +__CALLABLE_TYPES__ = ( FunctionType, MethodType, GeneratorType, CodeType, BuiltinFunctionType, BuiltinMethodType, ) + +# Always wrap these types without attempting to subclass +__WRAP_NO_SUBCLASS__ = ( ModuleType, XRangeType, SliceType, BufferType, TracebackType, FrameType, DictProxyType, + GetSetDescriptorType, MemberDescriptorType ) + __CALLABLE_TYPES__ + +# Don't wrap or sanitize. +__DONT_SANITIZE_TYPES__ = ( Number, bool, NoneType, NotImplementedType, EllipsisType, bytearray, ) + +# Don't wrap, but do sanitize. +__DONT_WRAP_TYPES__ = tuple() #( basestring, ) so that we can get the unsanitized string, we will now wrap basestring instances + +# Wrap contents, but not the container +__WRAP_SEQUENCES__ = ( tuple, list, ) +__WRAP_SETS__ = ( set, frozenset, ) +__WRAP_MAPPINGS__ = ( dict, UserDict, ) + + +# Define the set of characters that are not sanitized, and define a set of mappings for those that are. +# characters that are valid +VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) + +# characters that are allowed but need to be escaped +CHARACTER_MAP = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + +INVALID_CHARACTER = "X" + +def sanitize_lists_to_string( values, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP, invalid_character=INVALID_CHARACTER ): + return _sanitize_lists_to_string( values, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) + + +def wrap_with_safe_string( value, no_wrap_classes = None ): + """ + Recursively wrap values that should be wrapped. + """ + + def __do_wrap( value ): + if isinstance( value, SafeStringWrapper ): + # Only ever wrap one-layer + return value + if callable( value ): + safe_class = CallableSafeStringWrapper + else: + safe_class = SafeStringWrapper + if isinstance( value, no_wrap_classes ): + return value + if isinstance( value, __DONT_WRAP_TYPES__ ): + return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + if isinstance( value, __WRAP_NO_SUBCLASS__ ): + return safe_class( value, safe_string_wrapper_function = __do_wrap ) + for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: + if isinstance( value, this_type ): + return this_type( map( __do_wrap, value ) ) + for this_type in __WRAP_MAPPINGS__: + if isinstance( value, this_type ): + # Wrap both key and value + return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) + # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. + # This allows e.g. isinstance to continue to work. + try: + wrapped_class_name = value.__name__ + wrapped_class = value + except: + wrapped_class_name = value.__class__.__name__ + wrapped_class = value.__class__ + value_mod = inspect.getmodule( value ) + if value_mod: + wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) + wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) + do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) + do_wrap_func = __do_wrap + global_dict = globals() + if wrapped_class_name in global_dict: + # Check to see if we have created a wrapper for this class yet, if so, reuse + wrapped_class = global_dict.get( wrapped_class_name ) + do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) + else: + try: + wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) + except TypeError, e: + # Fail-safe for when a class cannot be dynamically subclassed. + log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) + wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) + if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): + # Save this wrapper for reuse and pickling/copying + global_dict[ wrapped_class_name ] = wrapped_class + do_wrap_func.__name__ = do_wrap_func_name + global_dict[ do_wrap_func_name ] = do_wrap_func + def pickle_safe_object( safe_object ): + return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) + # Set pickle and copy properties + copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) + return wrapped_class( value, safe_string_wrapper_function = do_wrap_func ) + # Determine classes not to wrap + if no_wrap_classes: + if not isinstance( no_wrap_classes, ( tuple, list ) ): + no_wrap_classes = [ no_wrap_classes ] + no_wrap_classes = list( no_wrap_classes ) + list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + else: + no_wrap_classes = list( __DONT_SANITIZE_TYPES__ ) + [ SafeStringWrapper ] + no_wrap_classes = tuple( set( sorted( no_wrap_classes, key=str ) ) ) + return __do_wrap( value ) + + +# N.B. refer to e.g. https://docs.python.org/2/reference/datamodel.html for information on Python's Data Model. + + +class SafeStringWrapper( object ): + """ + Class that wraps and sanitizes any provided value's attributes + that will attempt to be cast into a string. + + Attempts to mimic behavior of original class, including operands. + + To ensure proper handling of e.g. subclass checks, the *wrap_with_safe_string()* + method should be used. + + This wrapping occurs in a recursive/parasitic fashion, as all called attributes of + the originally wrapped object will also be wrapped and sanitized, unless the attribute + is of a type found in __DONT_SANITIZE_TYPES__ + __DONT_WRAP_TYPES__, where e.g. ~(strings + will still be sanitized, but not wrapped), and e.g. integers will have neither. + """ + __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized' + __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__] + + + def __new__( cls, *arg, **kwd ): + # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data + # that will be used when other + this (this + other is handled by __add__) + safe_string_wrapper_function = kwd.get( 'safe_string_wrapper_function', None) or wrap_with_safe_string + try: + return super( SafeStringWrapper, cls ).__new__( cls, sanitize_lists_to_string( arg[0], valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + except Exception, e: + log.warning( "Could not provide an argument to %s.__new__: %s; will try without arguments.", cls, e ) + return super( SafeStringWrapper, cls ).__new__( cls ) + + def __init__( self, value, safe_string_wrapper_function = wrap_with_safe_string ): + self.unsanitized = value + self.__safe_string_wrapper_function__ = safe_string_wrapper_function + + def __str__( self ): + return sanitize_lists_to_string( self.unsanitized, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) + + def __repr__( self ): + return "%s object at %x on: %s" % ( sanitize_lists_to_string( self.__class__.__name__, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ), id( self ), sanitize_lists_to_string( repr( self.unsanitized ), valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) ) + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __le__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized <= other + + def __eq__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized == other + + def __ne__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized != other + + def __gt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized > other + + def __ge__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized >= other + + def __lt__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.unsanitized < other + + def __cmp__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return cmp( self.unsanitized, other ) + + # Do not implement __rcmp__, python 2.2 < 2.6 + + def __hash__( self ): + return hash( self.unsanitized ) + + def __nonzero__( self ): + return bool( self.unsanitized ) + + # Do not implement __unicode__, we will rely on __str__ + + def __getattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + #FIXME: is this ever reached? + return object.__getattr__( self, name ) + return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) ) + + def __setattr__( self, name, value ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__setattr__( self, name, value ) + return setattr( self.unsanitized, name, value ) + + def __delattr__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__delattr__( self, name ) + return delattr( self.unsanitized, name ) + + def __getattribute__( self, name ): + if name in SafeStringWrapper.__NO_WRAP_NAMES__: + return object.__getattribute__( self, name ) + return self.__safe_string_wrapper_function__( getattr( object.__getattribute__( self, 'unsanitized' ), name ) ) + + # Skip Descriptors + + # Skip __slots__ + + # Don't need __metaclass__, we'll use the helper function to handle with subclassing for e.g. isinstance() + + # Revisit: + # __instancecheck__ + # __subclasscheck__ + # We are using a helper class to create dynamic subclasses to handle class checks + + # We address __call__ as needed based upon unsanitized, through the use of a CallableSafeStringWrapper class + + def __len__( self ): + original_value = self.unsanitized + while isinstance( original_value, SafeStringWrapper ): + original_value = self.unsanitized + return len( self.unsanitized ) + + def __getitem__( self, key ): + return self.__safe_string_wrapper_function__( self.unsanitized[ key ] ) + + def __setitem__( self, key, value ): + while isinstance( value, SafeStringWrapper ): + value = value.unsanitized + self.unsanitized[ key ] = value + + def __delitem__( self, key ): + del self.unsanitized[ key ] + + def __iter__( self ): + return iter( map( self.__safe_string_wrapper_function__, iter( self.unsanitized ) ) ) + + # Do not implement __reversed__ + + def __contains__( self, item ): + # FIXME: Do we need to consider if item is/isn't or does/doesn't contain SafeStringWrapper? + # When considering e.g. nested lists/dicts/etc, this gets complicated + while isinstance( item, SafeStringWrapper ): + item = item.unsanitized + return item in self.unsanitized + + # Not sure that we need these slice methods, but will provide anyway + def __getslice__( self, i, j ): + return self.__safe_string_wrapper_function__( self.unsanitized[ i:j ] ) + + def __setslice__( self, i, j, value ): + self.unsanitized[ i:j ] = value + + def __delslice__( self, i, j ): + del self.unsanitized[ i:j ] + + def __add__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized + other ) + + def __sub__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized - other ) + + def __mul__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized * other ) + + def __floordiv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized // other ) + + def __mod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized % other ) + + def __divmod__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( divmod( self.unsanitized, other ) ) + + def __pow__( self, *other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( self.unsanitized, *other ) ) + + def __lshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized << other ) + + def __rshift__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized >> other ) + + def __and__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized & other ) + + def __xor__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized ^ other ) + + def __or__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized | other ) + + def __div__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + def __truediv__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( self.unsanitized / other ) + + # The only reflected operand that we will define is __rpow__, due to coercion rules complications as per docs + def __rpow__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return self.__safe_string_wrapper_function__( pow( other, self.unsanitized ) ) + + # Do not implement in-place operands + + def __neg__( self ): + return __safe_string_wrapper_function__( -self.unsanitized ) + + def __pos__( self ): + return __safe_string_wrapper_function__( +self.unsanitized ) + + def __abs__( self ): + return __safe_string_wrapper_function__( abs( self.unsanitized ) ) + + def __invert__( self ): + return __safe_string_wrapper_function__( ~self.unsanitized ) + + def __complex__( self ): + return __safe_string_wrapper_function__( complex( self.unsanitized ) ) + + def __int__( self ): + return int( self.unsanitized ) + + def __float__( self ): + return float( self.unsanitized ) + + def __oct__( self ): + return oct( self.unsanitized ) + + def __hex__( self ): + return hex( self.unsanitized ) + + def __index__( self ): + return self.unsanitized.index() + + def __coerce__( self, other ): + while isinstance( other, SafeStringWrapper ): + other = other.unsanitized + return coerce( self.unsanitized, other ) + + def __enter__( self ): + return self.unsanitized.__enter__() + + def __exit__( self, *args ): + return self.unsanitized.__exit__( *args ) + +class CallableSafeStringWrapper( SafeStringWrapper ): + + def __call__( self, *args, **kwds ): + return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) + + +# Enable pickling/deepcopy +def pickle_SafeStringWrapper( safe_object ): + args = ( safe_object.unsanitized, ) + cls = SafeStringWrapper + if isinstance( safe_object, CallableSafeStringWrapper ): + cls = CallableSafeStringWrapper + return ( cls, args ) +copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) +copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) + https://bitbucket.org/galaxy/galaxy-central/commits/2e8dd2949dd3/ Changeset: 2e8dd2949dd3 Branch: stable User: natefoo Date: 2015-01-13 15:33:50+00:00 Summary: Merge next-stable to stable for release_2015.01.13 Affected #: 427 files diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee .hgignore --- a/.hgignore +++ b/.hgignore @@ -67,6 +67,7 @@ shed_data_manager_conf.xml object_store_conf.xml job_metrics_conf.xml +workflow_schedulers_conf.xml config/* static/welcome.html.* static/welcome.html diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee buildbot_setup.sh --- a/buildbot_setup.sh +++ b/buildbot_setup.sh @@ -126,5 +126,3 @@ echo "Appending tool-data/shared/ucsc/builds.txt.buildbot to tool-data/shared/ucsc/builds.txt" cat tool-data/shared/ucsc/builds.txt.buildbot >> tool-data/shared/ucsc/builds.txt - -python ./scripts/fetch_eggs.py all diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/galaxy.frame.js --- a/client/galaxy/scripts/galaxy.frame.js +++ b/client/galaxy/scripts/galaxy.frame.js @@ -1,9 +1,8 @@ // dependencies define(["galaxy.masthead", "mvc/ui/ui-frames"], function(mod_masthead, Frames) { -// frame manager -var GalaxyFrame = Backbone.View.extend( -{ +/** Frame manager uses the ui-frames to create the scratch book masthead icon and functionality **/ +var GalaxyFrame = Backbone.View.extend({ // base element el_main: 'body', @@ -17,8 +16,7 @@ button_load : null, // initialize - initialize : function(options) - { + initialize : function(options) { // add to masthead menu var self = this; @@ -28,8 +26,7 @@ }); // add activate icon - this.button_active = new mod_masthead.GalaxyMastheadIcon ( - { + this.button_active = new mod_masthead.GalaxyMastheadIcon({ icon : 'fa-th', tooltip : 'Enable/Disable Scratchbook', onclick : function() { self._activate(); }, @@ -44,8 +41,7 @@ Galaxy.masthead.append(this.button_active); // add load icon - this.button_load = new mod_masthead.GalaxyMastheadIcon ( - { + this.button_load = new mod_masthead.GalaxyMastheadIcon({ icon : 'fa-eye', tooltip : 'Show/Hide Scratchbook', onclick : function(e) { @@ -122,35 +118,77 @@ }); }, + + /** + * Add a trackster visualization to the frames. + */ + add_trackster_viz: function(viz_id) { + var self = this; + require(['viz/visualization', 'viz/trackster'], function(visualization, trackster) { + var viz = new visualization.Visualization({id: viz_id}); + $.when( viz.fetch() ).then( function() { + var ui = new trackster.TracksterUI(galaxy_config.root); + + // Construct frame config based on dataset's type. + var frame_config = { + title: viz.get('name'), + type: 'other', + content: function(parent_elt) { + // Create view config. + var view_config = { + container: parent_elt, + name: viz.get('title'), + id: viz.id, + // FIXME: this will not work with custom builds b/c the dbkey needed to be encoded. + dbkey: viz.get('dbkey'), + stand_alone: false + }, + latest_revision = viz.get('latest_revision'), + drawables = latest_revision.config.view.drawables; + + // Set up datasets in drawables. + _.each(drawables, function(d) { + d.dataset = { + hda_ldda: d.hda_ldda, + id: d.dataset_id + }; + }); + + view = ui.create_visualization(view_config, + latest_revision.config.viewport, + latest_revision.config.view.drawables, + latest_revision.config.bookmarks, + false); + } + }; + + self.add(frame_config); + }); + }); + }, /** * Add and display a new frame/window based on options. */ - add: function(options) - { + add: function(options){ // open new tab - if (options.target == '_blank') - { + if (options.target == '_blank'){ window.open(options.content); return; } // reload entire window - if (options.target == '_top' || options.target == '_parent' || options.target == '_self') - { + if (options.target == '_top' || options.target == '_parent' || options.target == '_self'){ window.location = options.content; return; } // validate - if (!this.active) - { + if (!this.active){ // fix url if main frame is unavailable var $galaxy_main = $(window.parent.document).find('#galaxy_main'); - if (options.target == 'galaxy_main' || options.target == 'center') - { - if ($galaxy_main.length === 0) - { + if (options.target == 'galaxy_main' || options.target == 'center'){ + if ($galaxy_main.length === 0){ var href = options.content; if (href.indexOf('?') == -1) href += '?'; @@ -173,11 +211,9 @@ }, // activate/disable panel - _activate: function () - { + _activate: function (){ // check - if (this.active) - { + if (this.active){ // disable this.active = false; @@ -196,8 +232,7 @@ }, // update frame counter - _refresh: function() - { + _refresh: function(){ // update on screen counter this.button_load.number(this.frames.length()); diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/galaxy.interactive_environments.js --- /dev/null +++ b/client/galaxy/scripts/galaxy.interactive_environments.js @@ -0,0 +1,59 @@ +/** + * Internal function to remove content from the main area and add the notebook. + * Not idempotent + */ +function append_notebook(url){ + clear_main_area(); + $('#main').append('<iframe frameBorder="0" seamless="seamless" style="width: 100%; height: 100%; overflow:hidden;" scrolling="no" src="'+ url +'"></iframe>' + ); +} + +function clear_main_area(){ + $('#spinner').remove(); + $('#main').children().remove(); +} + +function display_spinner(){ + $('#main').append('<img id="spinner" src="' + galaxy_root + '/static/style/largespinner.gif" style="position:absolute;margin:auto;top:0;left:0;right:0;bottom:0;">'); +} + + +/** + * Test availability of a URL, and call a callback when done. + * http://stackoverflow.com/q/25390206/347368 + * @param {String} url: URL to test availability of. Must return a 200 (302->200 is OK). + * @param {String} callback: function to call once successfully connected. + * + */ +function test_ie_availability(url, success_callback){ + var request_count = 0; + display_spinner(); + interval = setInterval(function(){ + $.ajax({ + url: url, + xhrFields: { + withCredentials: true + }, + type: "GET", + timeout: 500, + success: function(){ + console.log("Connected to IE, returning"); + clearInterval(interval); + success_callback(); + }, + error: function(jqxhr, status, error){ + request_count++; + console.log("Request " + request_count); + if(request_count > 30){ + clearInterval(interval); + clear_main_area(); + toastr.error( + "Could not connect to IE, contact your administrator", + "Error", + {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': false} + ); + } + } + }); + }, 1000); +} diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/galaxy.library.js --- a/client/galaxy/scripts/galaxy.library.js +++ b/client/galaxy/scripts/galaxy.library.js @@ -16,19 +16,19 @@ "mvc/library/library-library-view", "mvc/library/library-folder-view" ], -function(mod_masthead, - mod_utils, - mod_toastr, - mod_baseMVC, - mod_library_model, - mod_folderlist_view, - mod_librarylist_view, - mod_librarytoolbar_view, - mod_foldertoolbar_view, - mod_library_dataset_view, - mod_library_library_view, - mod_library_folder_view - ) { + function(mod_masthead, + mod_utils, + mod_toastr, + mod_baseMVC, + mod_library_model, + mod_folderlist_view, + mod_librarylist_view, + mod_librarytoolbar_view, + mod_foldertoolbar_view, + mod_library_dataset_view, + mod_library_library_view, + mod_library_folder_view + ) { // ============================================================================ // ROUTER @@ -36,40 +36,44 @@ initialize: function() { this.routesHit = 0; //keep count of number of routes handled by the application - Backbone.history.on('route', function() { this.routesHit++; }, this); - }, + Backbone.history.on( 'route', function() { this.routesHit++; }, this ); +}, - routes: { +routes: { "" : "libraries", + "page/:show_page" : "libraries_page", "library/:library_id/permissions" : "library_permissions", "folders/:folder_id/permissions" : "folder_permissions", "folders/:id" : "folder_content", + "folders/:id/page/:show_page" : "folder_page", "folders/:folder_id/datasets/:dataset_id" : "dataset_detail", "folders/:folder_id/datasets/:dataset_id/permissions" : "dataset_permissions", "folders/:folder_id/datasets/:dataset_id/versions/:ldda_id" : "dataset_version", "folders/:folder_id/download/:format" : "download", "folders/:folder_id/import/:source" : "import_datasets" - }, +}, - back: function() { - if(this.routesHit > 1) { +back: function() { + if( this.routesHit > 1 ) { //more than one route hit -> user did not land to current page directly window.history.back(); - } else { + } else { //otherwise go to the home page. Use replaceState if available so //the navigation doesn't create an extra history entry - this.navigate('#', {trigger:true, replace:true}); - } + this.navigate( '#', { trigger:true, replace:true } ); } +} }); // ============================================================================ /** session storage for library preferences */ var LibraryPrefs = mod_baseMVC.SessionStorageModel.extend({ defaults : { - with_deleted : false, - sort_order : 'asc', - sort_by : 'name' + with_deleted : false, + sort_order : 'asc', + sort_by : 'name', + library_page_size : 20, + folder_page_size : 15 } }); @@ -88,77 +92,97 @@ initialize : function(){ Galaxy.libraries = this; - this.preferences = new LibraryPrefs( {id: 'global-lib-prefs'} ); + this.preferences = new LibraryPrefs( { id: 'global-lib-prefs' } ); this.library_router = new LibraryRouter(); - this.library_router.on('route:libraries', function() { - Galaxy.libraries.libraryToolbarView = new mod_librarytoolbar_view.LibraryToolbarView(); - Galaxy.libraries.libraryListView = new mod_librarylist_view.LibraryListView(); + this.library_router.on( 'route:libraries', function() { + Galaxy.libraries.libraryToolbarView = new mod_librarytoolbar_view.LibraryToolbarView(); + Galaxy.libraries.libraryListView = new mod_librarylist_view.LibraryListView(); + }); + + this.library_router.on('route:libraries_page', function( show_page ) { + if ( Galaxy.libraries.libraryToolbarView === null ){ + Galaxy.libraries.libraryToolbarView = new mod_librarytoolbar_view.LibraryToolbarView(); + Galaxy.libraries.libraryListView = new mod_librarylist_view.LibraryListView( { show_page: show_page } ); + } else { + Galaxy.libraries.libraryListView.render( { show_page: show_page } ) + } }); - this.library_router.on('route:folder_content', function(id) { + this.library_router.on( 'route:folder_content', function( id ) { if (Galaxy.libraries.folderToolbarView){ - Galaxy.libraries.folderToolbarView.$el.unbind('click'); + Galaxy.libraries.folderToolbarView.$el.unbind( 'click' ); } - Galaxy.libraries.folderToolbarView = new mod_foldertoolbar_view.FolderToolbarView({id: id}); - Galaxy.libraries.folderListView = new mod_folderlist_view.FolderListView({id: id}); + Galaxy.libraries.folderToolbarView = new mod_foldertoolbar_view.FolderToolbarView( { id: id } ); + Galaxy.libraries.folderListView = new mod_folderlist_view.FolderListView( { id: id } ); }); - this.library_router.on('route:download', function(folder_id, format) { - if ($('#folder_list_body').find(':checked').length === 0) { - mod_toastr.info( 'You must select at least one dataset to download' ); - Galaxy.libraries.library_router.navigate('folders/' + folder_id, {trigger: true, replace: true}); - } else { - Galaxy.libraries.folderToolbarView.download(folder_id, format); - Galaxy.libraries.library_router.navigate('folders/' + folder_id, {trigger: false, replace: true}); - } + this.library_router.on( 'route:folder_page', function( id, show_page ) { + if ( Galaxy.libraries.folderToolbarView === null ){ + Galaxy.libraries.folderToolbarView = new mod_foldertoolbar_view.FolderToolbarView( {id: id} ); + Galaxy.libraries.folderListView = new mod_folderlist_view.FolderListView( { id: id, show_page: show_page } ); + } else { + Galaxy.libraries.folderListView.render( { id: id, show_page: parseInt( show_page ) } ) + } }); - this.library_router.on('route:dataset_detail', function(folder_id, dataset_id){ - if (Galaxy.libraries.datasetView){ - Galaxy.libraries.datasetView.$el.unbind('click'); - } - Galaxy.libraries.datasetView = new mod_library_dataset_view.LibraryDatasetView({id: dataset_id}); - }); - this.library_router.on('route:dataset_version', function(folder_id, dataset_id, ldda_id){ - if (Galaxy.libraries.datasetView){ - Galaxy.libraries.datasetView.$el.unbind('click'); - } - Galaxy.libraries.datasetView = new mod_library_dataset_view.LibraryDatasetView({id: dataset_id, ldda_id: ldda_id, show_version: true}); - }); + this.library_router.on( 'route:download', function( folder_id, format ) { + if ( $( '#folder_list_body' ).find( ':checked' ).length === 0 ) { + mod_toastr.info( 'You must select at least one dataset to download' ); + Galaxy.libraries.library_router.navigate( 'folders/' + folder_id, { trigger: true, replace: true } ); + } else { + Galaxy.libraries.folderToolbarView.download( folder_id, format ); + Galaxy.libraries.library_router.navigate( 'folders/' + folder_id, { trigger: false, replace: true } ); + } + }); - this.library_router.on('route:dataset_permissions', function(folder_id, dataset_id){ - if (Galaxy.libraries.datasetView){ - Galaxy.libraries.datasetView.$el.unbind('click'); - } - Galaxy.libraries.datasetView = new mod_library_dataset_view.LibraryDatasetView({id: dataset_id, show_permissions: true}); - }); + this.library_router.on( 'route:dataset_detail', function(folder_id, dataset_id){ + if (Galaxy.libraries.datasetView){ + Galaxy.libraries.datasetView.$el.unbind('click'); + } + Galaxy.libraries.datasetView = new mod_library_dataset_view.LibraryDatasetView({id: dataset_id}); + }); - this.library_router.on('route:library_permissions', function(library_id){ - if (Galaxy.libraries.libraryView){ - Galaxy.libraries.libraryView.$el.unbind('click'); - } - Galaxy.libraries.libraryView = new mod_library_library_view.LibraryView({id: library_id, show_permissions: true}); - }); + this.library_router.on( 'route:dataset_version', function(folder_id, dataset_id, ldda_id){ + if (Galaxy.libraries.datasetView){ + Galaxy.libraries.datasetView.$el.unbind('click'); + } + Galaxy.libraries.datasetView = new mod_library_dataset_view.LibraryDatasetView({id: dataset_id, ldda_id: ldda_id, show_version: true}); + }); - this.library_router.on('route:folder_permissions', function(folder_id){ - if (Galaxy.libraries.folderView){ - Galaxy.libraries.folderView.$el.unbind('click'); - } - Galaxy.libraries.folderView = new mod_library_folder_view.FolderView({id: folder_id, show_permissions: true}); - }); - this.library_router.on('route:import_datasets', function(folder_id, source){ - if (Galaxy.libraries.folderToolbarView && Galaxy.libraries.folderListView){ - Galaxy.libraries.folderToolbarView.showImportModal({source:source}); - } else { - Galaxy.libraries.folderToolbarView = new mod_foldertoolbar_view.FolderToolbarView({id: folder_id}); - Galaxy.libraries.folderListView = new mod_folderlist_view.FolderListView({id: folder_id}); - Galaxy.libraries.folderToolbarView.showImportModal({source: source}); - } - }); + this.library_router.on( 'route:dataset_permissions', function(folder_id, dataset_id){ + if (Galaxy.libraries.datasetView){ + Galaxy.libraries.datasetView.$el.unbind('click'); + } + Galaxy.libraries.datasetView = new mod_library_dataset_view.LibraryDatasetView({id: dataset_id, show_permissions: true}); + }); - Backbone.history.start({pushState: false}); + this.library_router.on( 'route:library_permissions', function(library_id){ + if (Galaxy.libraries.libraryView){ + Galaxy.libraries.libraryView.$el.unbind('click'); + } + Galaxy.libraries.libraryView = new mod_library_library_view.LibraryView({id: library_id, show_permissions: true}); + }); + + this.library_router.on( 'route:folder_permissions', function(folder_id){ + if (Galaxy.libraries.folderView){ + Galaxy.libraries.folderView.$el.unbind('click'); + } + Galaxy.libraries.folderView = new mod_library_folder_view.FolderView({id: folder_id, show_permissions: true}); + }); + + this.library_router.on( 'route:import_datasets', function( folder_id, source ){ + if ( Galaxy.libraries.folderToolbarView && Galaxy.libraries.folderListView ){ + Galaxy.libraries.folderToolbarView.showImportModal( { source:source } ); + } else { + Galaxy.libraries.folderToolbarView = new mod_foldertoolbar_view.FolderToolbarView( { id: folder_id } ); + Galaxy.libraries.folderListView = new mod_folderlist_view.FolderListView( { id: folder_id } ); + Galaxy.libraries.folderToolbarView.showImportModal( { source: source } ); + } + }); + + Backbone.history.start({pushState: false}); } }); diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/galaxy.masthead.js --- a/client/galaxy/scripts/galaxy.masthead.js +++ b/client/galaxy/scripts/galaxy.masthead.js @@ -1,9 +1,8 @@ // dependencies define([], function() { -// masthead -var GalaxyMasthead = Backbone.View.extend( -{ +/** Masthead **/ +var GalaxyMasthead = Backbone.View.extend({ // base element el_masthead: '#everything', @@ -17,8 +16,7 @@ list: [], // initialize - initialize : function(options) - { + initialize : function(options) { // update options this.options = options; @@ -51,27 +49,23 @@ }, // configure events - events: - { + events: { 'click' : '_click', 'mousedown' : function(e) { e.preventDefault() } }, // adds a new item to the masthead - append : function(item) - { + append : function(item) { return this._add(item, true); }, // adds a new item to the masthead - prepend : function(item) - { + prepend : function(item) { return this._add(item, false); }, // activate - highlight: function(id) - { + highlight: function(id) { var current = $(this.el).find('#' + id + '> li'); if (current) { current.addClass('active'); @@ -79,11 +73,9 @@ }, // adds a new item to the masthead - _add : function(item, append) - { + _add : function(item, append) { var $loc = $(this.el).find('#' + item.location); - if ($loc) - { + if ($loc){ // create frame for new item var $current = $(item.el); @@ -106,8 +98,7 @@ }, // handle click event - _click: function(e) - { + _click: function(e) { // close all popups var $all = $(this.el).find('.popup'); if ($all) { @@ -129,8 +120,7 @@ */ // fill template - _template: function(options) - { + _template: function(options) { var brand_text = options.brand ? ("/ " + options.brand) : "" ; return '<div><div id="masthead" class="navbar navbar-fixed-top navbar-inverse">' + '<div style="position: relative; right: -50%; float: left;">' + @@ -150,12 +140,10 @@ } }); -// icon -var GalaxyMastheadIcon = Backbone.View.extend( -{ +/** Masthead icon **/ +var GalaxyMastheadIcon = Backbone.View.extend({ // icon options - options: - { + options:{ id : '', icon : 'fa-cog', tooltip : '', @@ -169,8 +157,7 @@ location: 'iconbar', // initialize - initialize: function (options) - { + initialize: function (options){ // read in defaults if (options) this.options = _.defaults(options, this.options); @@ -189,20 +176,17 @@ }, // show - show: function() - { + show: function(){ $(this.el).css({visibility : 'visible'}); }, // show - hide: function() - { + hide: function(){ $(this.el).css({visibility : 'hidden'}); }, // switch icon - icon: function (new_icon) - { + icon: function (new_icon){ // update icon class $(this.el).find('.icon').removeClass(this.options.icon) .addClass(new_icon); @@ -212,26 +196,22 @@ }, // toggle - toggle: function() - { + toggle: function(){ $(this.el).addClass('toggle'); }, // untoggle - untoggle: function() - { + untoggle: function(){ $(this.el).removeClass('toggle'); }, // set/get number - number: function(new_number) - { + number: function(new_number){ $(this.el).find('.number').text(new_number); }, // fill template icon - _template: function (options) - { + _template: function (options){ var tmpl = '<div id="' + options.id + '" class="symbol">' + '<div class="icon fa fa-2x ' + options.icon + '"></div>'; if (options.with_number) @@ -243,12 +223,10 @@ } }); -// tab -var GalaxyMastheadTab = Backbone.View.extend( -{ +/** Masthead tab **/ +var GalaxyMastheadTab = Backbone.View.extend({ // main options - options: - { + options:{ id : '', title : '', target : '_parent', @@ -268,52 +246,50 @@ $menu: null, // events - events: - { + events:{ 'click .head' : '_head' }, // initialize - initialize: function (options) - { + initialize: function ( options ){ // read in defaults - if (options) - this.options = _.defaults(options, this.options); - + if ( options ){ + this.options = _.defaults( options, this.options ); + } + // update url - if (this.options.content && this.options.content.indexOf('//') === -1) + if ( this.options.content !== undefined && this.options.content.indexOf( '//' ) === -1 ){ this.options.content = galaxy_config.root + this.options.content; + } // add template for tab - this.setElement($(this._template(this.options))); + this.setElement( $( this._template( this.options ) ) ); // disable menu items that are not available to anonymous user // also show title to explain why they are disabled - if (this.options.disabled){ - $(this.el).find('.root').addClass('disabled'); + if ( this.options.disabled ){ + $( this.el ).find( '.root' ).addClass( 'disabled' ); this._attachPopover(); } // visiblity - if (!this.options.visible) + if ( !this.options.visible ){ this.hide(); + } }, // show - show: function() - { + show: function(){ $(this.el).css({visibility : 'visible'}); }, // show - hide: function() - { + hide: function(){ $(this.el).css({visibility : 'hidden'}); }, // add menu item - add: function (options) - { + add: function (options){ // menu option defaults var menuOptions = { title : 'Title', @@ -333,8 +309,7 @@ menuOptions.content = galaxy_config.root + menuOptions.content; // check if submenu element is available - if (!this.$menu) - { + if (!this.$menu){ // insert submenu element into root $(this.el).find('.root').append(this._templateMenu()); @@ -353,8 +328,7 @@ // add events var self = this; - $item.on('click', function(e) - { + $item.on('click', function(e){ // prevent default e.preventDefault(); @@ -372,8 +346,7 @@ }, // show menu on header click - _head: function(e) - { + _head: function(e){ // prevent default e.preventDefault(); @@ -387,12 +360,11 @@ } }, - _attachPopover : function() - { + _attachPopover : function(){ var $popover_element = $(this.el).find('.head'); $popover_element.popover({ html: true, - content: 'Please <a href="/user/login">log in</a> or <a href="/user/create">register</a> to use this feature.', + content: 'Please <a href="' + galaxy_config.root + '/user/login">log in</a> or <a href="' + galaxy_config.root + '/user/create">register</a> to use this feature.', placement: 'bottom' }).on('shown.bs.popover', function() { // hooking on bootstrap event to automatically hide popovers after delay setTimeout(function() { @@ -402,25 +374,21 @@ }, // fill template header - _templateMenuItem: function (options) - { + _templateMenuItem: function (options){ return '<li><a href="' + options.content + '" target="' + options.target + '">' + options.title + '</a></li>'; }, // fill template header - _templateMenu: function () - { + _templateMenu: function (){ return '<ul class="popup dropdown-menu"></ul>'; }, - _templateDivider: function() - { + _templateDivider: function(){ return '<li class="divider"></li>'; }, // fill template - _template: function (options) - { + _template: function (options){ // start template var tmpl = '<ul id="' + options.id + '" class="nav navbar-nav" border="0" cellspacing="0">' + '<li class="root dropdown" style="">' + diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/galaxy.menu.js --- a/client/galaxy/scripts/galaxy.menu.js +++ b/client/galaxy/scripts/galaxy.menu.js @@ -1,13 +1,8 @@ -/* - galaxy menu -*/ - // dependencies define(["galaxy.masthead"], function(mod_masthead) { -// frame manager -var GalaxyMenu = Backbone.Model.extend( -{ +/** GalaxyMenu uses the GalaxyMasthead class in order to add menu items and icons to the Masthead **/ +var GalaxyMenu = Backbone.Model.extend({ // options options: null, @@ -15,23 +10,21 @@ masthead: null, // initialize - initialize: function(options) - { + initialize: function(options) { this.options = options.config; this.masthead = options.masthead; this.create(); }, // default menu - create: function() - { + create: function(){ // // Analyze data tab. // var tab_analysis = new mod_masthead.GalaxyMastheadTab({ id : "analysis", title : "Analyze Data", - content : "root/index", + content : "", title_attribute : 'Analysis home view' }); this.masthead.append(tab_analysis); @@ -137,8 +130,7 @@ var tab_visualization = new mod_masthead.GalaxyMastheadTab(visualization_options); - if (this.options.user.valid) //add submenu only when user is logged in - { + if (this.options.user.valid){ //add submenu only when user is logged in tab_visualization.add({ title : "New Track Browser", content : "visualization/trackster", @@ -155,8 +147,7 @@ // // Cloud menu. // - if (this.options.enable_cloud_launch) - { + if (this.options.enable_cloud_launch){ var tab_cloud = new mod_masthead.GalaxyMastheadTab({ id : "cloud", title : "Cloud", @@ -172,12 +163,11 @@ // // Admin. // - if (this.options.is_admin_user) - { + if (this.options.is_admin_user) { var tab_admin = new mod_masthead.GalaxyMastheadTab({ id : "admin", title : "Admin", - content : "admin/index", + content : "admin", extra_class : "admin-only", title_attribute : 'Administer this Galaxy' }); @@ -192,8 +182,7 @@ title : "Help", title_attribute : 'Support, contact, and community hubs' }); - if (this.options.biostar_url) - { + if (this.options.biostar_url){ tab_help.add({ title : "Galaxy Biostar", content : this.options.biostar_url_redirect, @@ -235,8 +224,7 @@ content : this.options.citation_url, target : "_blank" }); - if (this.options.terms_url) - { + if (this.options.terms_url){ tab_help.add({ title : "Terms and Conditions", content : this.options.terms_url, @@ -248,8 +236,7 @@ // // User tab. // - if (!this.options.user.valid) - { + if (!this.options.user.valid){ var tab_user = new mod_masthead.GalaxyMastheadTab({ id : "user", title : "User", @@ -265,8 +252,7 @@ }); // register - if (this.options.allow_user_creation) - { + if (this.options.allow_user_creation){ tab_user.add({ title : "Register", content : "user/create", @@ -331,8 +317,7 @@ target : "galaxy_main" }); - if (this.options.use_remote_user) - { + if (this.options.use_remote_user){ tab_user.add({ title : "Public Name", content : "user/edit_username?cntrller=user", diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/base-mvc.js --- a/client/galaxy/scripts/mvc/base-mvc.js +++ b/client/galaxy/scripts/mvc/base-mvc.js @@ -380,11 +380,6 @@ /** allow the view to be dragged, set up event handlers */ draggableOn : function(){ this.draggable = true; - //TODO: I have no idea why this doesn't work with the events hash or jq.on()... - //this.$el.find( '.title-bar' ) - // .attr( 'draggable', true ) - // .bind( 'dragstart', this.dragStartHandler, false ) - // .bind( 'dragend', this.dragEndHandler, false ); this.dragStartHandler = _.bind( this._dragStartHandler, this ); this.dragEndHandler = _.bind( this._dragEndHandler, this ); @@ -402,23 +397,22 @@ }, /** sets the dataTransfer data to the model's toJSON - * @fires dragstart (bbone event) which is passed this view + * @fires draggable:dragstart (bbone event) which is passed the event and this view */ _dragStartHandler : function( event ){ - //this.debug( 'dragStartHandler:', this, event, arguments ) - this.trigger( 'dragstart', this ); event.dataTransfer.effectAllowed = 'move'; + //ASSUMES: this.model //TODO: all except IE: should be 'application/json', IE: must be 'text' event.dataTransfer.setData( 'text', JSON.stringify( this.model.toJSON() ) ); + this.trigger( 'draggable:dragstart', event, this ); return false; }, /** handle the dragend - * @fires dragend (bbone event) which is passed this view + * @fires draggable:dragend (bbone event) which is passed the event and this view */ _dragEndHandler : function( event ){ - this.trigger( 'dragend', this ); - //this.debug( 'dragEndHandler:', event ) + this.trigger( 'draggable:dragend', event, this ); return false; } }; diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/collection/collection-li.js --- a/client/galaxy/scripts/mvc/collection/collection-li.js +++ b/client/galaxy/scripts/mvc/collection/collection-li.js @@ -100,8 +100,6 @@ /** add the DCE class to the list item */ className : ListItemView.prototype.className + " dataset-collection-element", - /** jq fx speed for this view */ - fxSpeed : 'fast', /** set up */ initialize : function( attributes ){ diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/dataset/dataset-li-edit.js --- a/client/galaxy/scripts/mvc/dataset/dataset-li-edit.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-li-edit.js @@ -181,6 +181,8 @@ } var $visualizations = $( this.templates.visualizations( visualizations, this ) ); + //HACK: need to re-write those directed at galaxy_main with linkTarget + $visualizations.find( '[target="galaxy_main"]').attr( 'target', this.linkTarget ); // use addBack here to include the root $visualizations elem (for the case of 1 visualization) this._addScratchBookFn( $visualizations.find( '.visualization-link' ).addBack( '.visualization-link' ) ); return $visualizations; @@ -188,6 +190,7 @@ /** add scratchbook functionality to visualization links */ _addScratchBookFn : function( $links ){ + var li = this; $links.click( function( ev ){ if( Galaxy.frame && Galaxy.frame.active ){ Galaxy.frame.add({ diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/dataset/dataset-li.js --- a/client/galaxy/scripts/mvc/dataset/dataset-li.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-li.js @@ -160,7 +160,7 @@ // add frame manager option onclick event var self = this; displayBtnData.onclick = function( ev ){ - if( Galaxy.frame && Galaxy.frame.active ){ + if (Galaxy.frame && Galaxy.frame.active) { // Add dataset to frames. Galaxy.frame.add_dataset(self.model.get('id')); ev.preventDefault(); diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/dataset/dataset-model.js --- a/client/galaxy/scripts/mvc/dataset/dataset-model.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-model.js @@ -363,74 +363,11 @@ Backbone.Collection.prototype.set.call( this, models, options ); }, -// /** Convert this ad-hoc collection of hdas to a formal collection tracked -// by the server. -// **/ -// promoteToHistoryDatasetCollection : function _promote( history, collection_type, options ){ -////TODO: seems like this would be better in mvc/collections -// options = options || {}; -// options.url = this.url(); -// options.type = "POST"; -// var full_collection_type = collection_type; -// var element_identifiers = [], -// name = null; -// -// // This mechanism is rough - no error handling, allows invalid selections, no way -// // for user to pick/override element identifiers. This is only really meant -// if( collection_type === "list" ) { -// this.chain().each( function( hda ) { -// // TODO: Handle duplicate names. -// var name = hda.attributes.name; -// var id = hda.get('id'); -// var content_type = hda.attributes.history_content_type; -// if( content_type === "dataset" ) { -// if( full_collection_type !== "list" ) { -// this.log( "Invalid collection type" ); -// } -// element_identifiers.push( { name: name, src: "hda", id: id } ); -// } else { -// if( full_collection_type === "list" ) { -// full_collection_type = "list:" + hda.attributes.collection_type; -// } else { -// if( full_collection_type !== "list:" + hda.attributes.collection_type ) { -// this.log( "Invalid collection type" ); -// } -// } -// element_identifiers.push( { name: name, src: "hdca", id: id } ); -// } -// }); -// name = "New Dataset List"; -// } else if( collection_type === "paired" ) { -// var ids = this.ids(); -// if( ids.length !== 2 ){ -// // TODO: Do something... -// } -// element_identifiers.push( { name: "forward", src: "hda", id: ids[ 0 ] } ); -// element_identifiers.push( { name: "reverse", src: "hda", id: ids[ 1 ] } ); -// name = "New Dataset Pair"; -// } -// options.data = { -// type: "dataset_collection", -// name: name, -// collection_type: full_collection_type, -// element_identifiers: JSON.stringify( element_identifiers ) -// }; -// -// var xhr = jQuery.ajax( options ); -// xhr.done( function( message, status, responseObj ){ -// history.refresh( ); -// }); -// xhr.fail( function( xhr, status, message ){ -// if( xhr.responseJSON && xhr.responseJSON.error ){ -// error = xhr.responseJSON.error; -// } else { -// error = xhr.responseJSON; -// } -// xhr.responseText = error; -// // Do something? -// }); -// return xhr; -// }, + ///** Convert this ad-hoc collection of hdas to a formal collection tracked + // by the server. + //**/ + //promoteToHistoryDatasetCollection : function _promote( history, collection_type, options ){ + //}, /** String representation. */ toString : function(){ diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/history/history-contents.js --- a/client/galaxy/scripts/mvc/history/history-contents.js +++ b/client/galaxy/scripts/mvc/history/history-contents.js @@ -187,6 +187,7 @@ /** copy an existing, accessible hda into this collection */ copy : function( id ){ +//TODO: incorp collections var collection = this, xhr = jQuery.post( this.url(), { source : 'hda', diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/history/history-model.js --- a/client/galaxy/scripts/mvc/history/history-model.js +++ b/client/galaxy/scripts/mvc/history/history-model.js @@ -14,7 +14,7 @@ * @constructs */ var History = Backbone.Model.extend( BASE_MVC.LoggableMixin ).extend( -/** @lends History.prototype */{ + BASE_MVC.mixin( BASE_MVC.SearchableModelMixin, /** @lends History.prototype */{ /** logger used to record this.log messages, commonly set to console */ //logger : console, @@ -117,7 +117,19 @@ return _.reduce( _.values( this.get( 'state_details' ) ), function( memo, num ){ return memo + num; }, 0 ); }, - // ........................................................................ ajax + // ........................................................................ search + /** What model fields to search with */ + searchAttributes : [ + 'name', 'annotation', 'tags' + ], + + /** Adding title and singular tag */ + searchAliases : { + title : 'name', + tag : 'tags' + }, + + // ........................................................................ updates /** does the contents collection indicate they're still running and need to be updated later? * delay + update if needed * @param {Function} onReadyCallback function to run when all contents are in the ready state @@ -187,6 +199,18 @@ return xhr; }, + // ........................................................................ ajax + /** save this history, _Mark_ing it as deleted (just a flag) */ + _delete : function( options ){ + if( this.get( 'deleted' ) ){ return jQuery.when(); } + return this.save( { deleted: true }, options ); + }, + /** save this history, _Mark_ing it as undeleted */ + undelete : function( options ){ + if( !this.get( 'deleted' ) ){ return jQuery.when(); } + return this.save( { deleted: false }, options ); + }, + /** Make a copy of this history on the server * @param {Boolean} current if true, set the copy as the new current history (default: true) * @param {String} name name of new history (default: none - server sets to: Copy of <current name>) @@ -210,9 +234,28 @@ //TODO:?? all datasets? var history = this, - xhr = jQuery.post( this.urlRoot, postData ); - xhr.done( function( newData ){ - history.trigger( 'copied', history, newData ); + copy = jQuery.post( this.urlRoot, postData ); + // if current - queue to setAsCurrent before firing 'copied' + if( current ){ + return copy.then( function( response ){ + var newHistory = new History( response ); + return newHistory.setAsCurrent() + .done( function(){ + history.trigger( 'copied', history, response ); + }); + }); + } + return copy.done( function( response ){ + history.trigger( 'copied', history, response ); + }); + }, + + setAsCurrent : function(){ + var history = this, + xhr = jQuery.getJSON( '/history/set_as_current?id=' + this.id ); + + xhr.done( function(){ + history.trigger( 'set-as-current', history ); }); return xhr; }, @@ -221,7 +264,7 @@ toString : function(){ return 'History(' + this.get( 'id' ) + ',' + this.get( 'name' ) + ')'; } -}); +})); //------------------------------------------------------------------------------ CLASS VARS /** When the history has running hdas, @@ -351,12 +394,15 @@ create : function create( data, hdas, historyOptions, xhrOptions ){ var collection = this, - history = new History( data || {}, hdas || [], historyOptions || {} ); - return history.save( xhrOptions ).done( function( newData ){ + xhr = jQuery.getJSON( galaxy_config.root + 'history/create_new_current' ); + return xhr.done( function( newData ){ + var history = new History( newData, [], historyOptions || {} ); // new histories go in the front //TODO: (implicit ordering by update time...) collection.unshift( history ); + collection.trigger( 'new-current' ); }); +//TODO: move back to using history.save (via Deferred.then w/ set_as_current) }, toString: function toString(){ diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/history/history-panel-edit-current.js --- a/client/galaxy/scripts/mvc/history/history-panel-edit-current.js +++ b/client/galaxy/scripts/mvc/history/history-panel-edit-current.js @@ -164,12 +164,13 @@ _setUpCollectionListeners : function(){ _super.prototype._setUpCollectionListeners.call( this ); + //TODO:?? may not be needed? see history-panel-edit, 369 // if a hidden item is created (gen. by a workflow), moves thru the updater to the ready state, // then: remove it from the collection if the panel is set to NOT show hidden datasets this.collection.on( 'state:ready', function( model, newState, oldState ){ if( ( !model.get( 'visible' ) ) && ( !this.storage.get( 'show_hidden' ) ) ){ - this.removeItemView( this.viewFromModel( model ) ); + this.removeItemView( model ); } }, this ); }, @@ -226,7 +227,7 @@ '</a>' ].join('') ); $emptyMsg.find( '.uploader-link' ).click( function( ev ){ - Galaxy.upload._eventShow( ev ); + Galaxy.upload.show( ev ); }); $emptyMsg.find( '.get-data-link' ).click( function( ev ){ $toolMenu.parent().scrollTop( 0 ); @@ -288,51 +289,23 @@ }, // ------------------------------------------------------------------------ sub-views - // reverse HID order - /** Override to reverse order of views - newest contents on top - * and add the current-content highlight class to currentContentId's view - */ + /** Override to add the current-content highlight class to currentContentId's view */ _attachItems : function( $whereTo ){ - var panel = this; - this.$list( $whereTo ).append( this.views.reverse().map( function( view ){ - // add current content - if( panel.currentContentId && view.model.id === panel.currentContentId ){ - panel.setCurrentContent( view ); - } - return view.$el; - })); + _super.prototype._attachItems.call( this, $whereTo ); + var panel = this, + currentContentView; + if( panel.currentContentId + && ( currentContentView = panel.viewFromModelId( panel.currentContentId ) ) ){ + panel.setCurrentContent( currentContentView ); + } return this; }, - /** Override to add datasets at the top */ + /** Override to remove any drill down panels */ addItemView : function( model, collection, options ){ - this.log( this + '.addItemView:', model ); - var panel = this; - if( !panel._filterItem( model ) ){ return undefined; } -//TODO: alternately, call collapse drilldown - // if this panel is currently hidden, return undefined - if( panel.panelStack.length ){ return this._collapseDrilldownPanel(); } - - var view = panel._createItemView( model ); - // use unshift and prepend to preserve reversed order - panel.views.unshift( view ); - - panel.scrollToTop(); - $({}).queue([ - function fadeOutEmptyMsg( next ){ - var $emptyMsg = panel.$emptyMessage(); - if( $emptyMsg.is( ':visible' ) ){ - $emptyMsg.fadeOut( panel.fxSpeed, next ); - } else { - next(); - } - }, - function createAndPrepend( next ){ - // render as hidden then slide down - panel.$list().prepend( view.render( 0 ).$el.hide() ); - view.$el.slideDown( panel.fxSpeed ); - } - ]); + var view = _super.prototype.addItemView.call( this, model, collection, options ); + if( !view ){ return view; } + if( this.panelStack.length ){ return this._collapseDrilldownPanel(); } return view; }, @@ -423,22 +396,6 @@ if( !msg.is( ':hidden' ) ){ msg.slideUp( this.fxSpeed ); } }, -//TODO: move show_deleted/hidden into panel from opt menu and remove this - /** add listeners to an external options menu (templates/webapps/galaxy/root/index.mako) */ - connectToOptionsMenu : function( optionsMenu ){ - if( !optionsMenu ){ - return this; - } - // set a visible indication in the popupmenu for show_hidden/deleted based on the currHistoryPanel's settings - this.on( 'new-storage', function( storage, panel ){ - if( optionsMenu && storage ){ - optionsMenu.findItemByHtml( _l( 'Include Deleted Datasets' ) ).checked = storage.get( 'show_deleted' ); - optionsMenu.findItemByHtml( _l( 'Include Hidden Datasets' ) ).checked = storage.get( 'show_hidden' ); - } - }); - return this; - }, - /** Return a string rep of the history */ toString : function(){ diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/history/history-panel-edit.js --- a/client/galaxy/scripts/mvc/history/history-panel-edit.js +++ b/client/galaxy/scripts/mvc/history/history-panel-edit.js @@ -69,7 +69,18 @@ this.multiselectActions = attributes.multiselectActions || this._getActions(); }, - // ------------------------------------------------------------------------ panel rendering + /** Override to handle history as drag-drop target */ + _setUpListeners : function(){ + _super.prototype._setUpListeners.call( this ); + + this.on( 'drop', function( ev, data ){ + this.dataDropped( data ); + // remove the drop target + this.dropTargetOff(); + }); + }, + + // ------------------------------------------------------------------------ listeners /** listening for collection events */ _setUpCollectionListeners : function(){ _super.prototype._setUpCollectionListeners.call( this ); @@ -104,6 +115,42 @@ return $newRender; }, + /** override to render counts when the items are rendered */ + renderItems : function( $whereTo ){ + var views = _super.prototype.renderItems.call( this, $whereTo ); + this._renderCounts( $whereTo ); + return views; + }, + + /** override to show counts, what's deleted/hidden, and links to toggle those */ + _renderCounts : function( $whereTo ){ +//TODO: too complicated + function toggleLink( _class, text ){ + return [ '<a class="', _class, '" href="javascript:void(0);">', text, '</a>' ].join( '' ); + } + $whereTo = $whereTo || this.$el; + var deleted = this.collection.where({ deleted: true }), + hidden = this.collection.where({ visible: false }), + msgs = []; + + if( this.views.length ){ + msgs.push( [ this.views.length, _l( 'shown' ) ].join( ' ' ) ); + } + if( deleted.length ){ + msgs.push( ( !this.showDeleted )? + ([ deleted.length, toggleLink( 'toggle-deleted-link', _l( 'deleted' ) ) ].join( ' ' )) + :( toggleLink( 'toggle-deleted-link', _l( 'hide deleted' ) ) ) + ); + } + if( hidden.length ){ + msgs.push( ( !this.showHidden )? + ([ hidden.length, toggleLink( 'toggle-hidden-link', _l( 'hidden' ) ) ].join( ' ' )) + :( toggleLink( 'toggle-hidden-link', _l( 'hide hidden' ) ) ) + ); + } + return $whereTo.find( '> .controls .subtitle' ).html( msgs.join( ', ' ) ); + }, + /** render the tags sub-view controller */ _renderTags : function( $where ){ var panel = this; @@ -266,6 +313,24 @@ }, // ------------------------------------------------------------------------ sub-views + // reverse HID order + /** Override to reverse order of views - newest contents on top */ + _attachItems : function( $whereTo ){ + this.$list( $whereTo ).append( this.views.reverse().map( function( view ){ + return view.$el; + })); + return this; + }, + + /** Override to add new contents at the top */ + _attachView : function( view ){ + var panel = this; + // override to control where the view is added, how/whether it's rendered + panel.views.unshift( view ); + panel.$list().prepend( view.render( 0 ).$el.hide() ); + view.$el.slideDown( panel.fxSpeed ); + }, + /** In this override, add purgeAllowed and whether tags/annotation editors should be shown */ _getItemViewOptions : function( model ){ var options = _super.prototype._getItemViewOptions.call( this, model ); @@ -278,22 +343,31 @@ return options; }, + ///** Override to alter data in drag based on multiselection */ + //_setUpItemViewListeners : function( view ){ + // var panel = this; + // _super.prototype._setUpItemViewListeners.call( panel, view ); + // + //}, + /** If this item is deleted and we're not showing deleted items, remove the view * @param {Model} the item model to check */ _handleHdaDeletionChange : function( itemModel ){ - if( itemModel.get( 'deleted' ) && !this.storage.get( 'show_deleted' ) ){ + if( itemModel.get( 'deleted' ) && !this.showDeleted ){ this.removeItemView( itemModel ); } + this._renderCounts(); }, /** If this item is hidden and we're not showing hidden items, remove the view * @param {Model} the item model to check */ _handleHdaVisibleChange : function( itemModel ){ - if( itemModel.hidden() && !this.storage.get( 'show_hidden' ) ){ + if( itemModel.hidden() && !this.storage.showHidden ){ this.removeItemView( itemModel ); } + this._renderCounts(); }, /** toggle the visibility of each content's tagsEditor applying all the args sent to this function */ @@ -319,7 +393,9 @@ // ------------------------------------------------------------------------ panel events /** event map */ events : _.extend( _.clone( _super.prototype.events ), { - 'click .show-selectors-btn' : 'toggleSelectors' + 'click .show-selectors-btn' : 'toggleSelectors', + 'click .toggle-deleted-link' : function( ev ){ this.toggleShowDeleted(); }, + 'click .toggle-hidden-link' : function( ev ){ this.toggleShowHidden(); } }), /** Update the history size display (curr. upper right of panel). @@ -328,6 +404,132 @@ this.$el.find( '.history-size' ).text( this.model.get( 'nice_size' ) ); }, + // ------------------------------------------------------------------------ as drop target + /** */ + dropTargetOn : function(){ + if( this.dropTarget ){ return this; } + this.dropTarget = true; + + //TODO: to init + var dropHandlers = { + 'dragenter' : _.bind( this.dragenter, this ), + 'dragover' : _.bind( this.dragover, this ), + 'dragleave' : _.bind( this.dragleave, this ), + 'drop' : _.bind( this.drop, this ) + }; +//TODO: scroll to top + var $dropTarget = this._renderDropTarget(); + this.$list().before([ this._renderDropTargetHelp(), $dropTarget ]); + for( var evName in dropHandlers ){ + if( dropHandlers.hasOwnProperty( evName ) ){ + //console.debug( evName, dropHandlers[ evName ] ); + $dropTarget.on( evName, dropHandlers[ evName ] ); + } + } + return this; + }, + + /** */ + _renderDropTarget : function(){ + return $( '<div/>' ).addClass( 'history-drop-target' ) + .css({ + 'height': '64px', + 'margin': '0px 10px 10px 10px', + 'border': '1px dashed black', + 'border-radius' : '3px' + }); + }, + + /** */ + _renderDropTargetHelp : function(){ + return $( '<div/>' ).addClass( 'history-drop-target-help' ) + .css({ + 'margin' : '10px 10px 4px 10px', + 'color' : 'grey', + 'font-size' : '80%', + 'font-style' : 'italic' + }) + .text( _l( 'Drag datasets here to copy them to the current history' ) ); + }, + + /** */ + dropTargetOff : function(){ + if( !this.dropTarget ){ return this; } + //this.log( 'dropTargetOff' ); + this.dropTarget = false; + // + //var dropTarget = this.$( '.history-drop-target' ).get(0); + //for( var evName in this._dropHandlers ){ + // if( this._dropHandlers.hasOwnProperty( evName ) ){ + // console.debug( evName, this._dropHandlers[ evName ] ); + // dropTarget.off( evName, this._dropHandlers[ evName ] ); + // } + //} + this.$( '.history-drop-target' ).remove(); + this.$( '.history-drop-target-help' ).remove(); + return this; + }, + /** */ + dropTargetToggle : function(){ + if( this.dropTarget ){ + this.dropTargetOff(); + } else { + this.dropTargetOn(); + } + return this; + }, + + /** */ + dragenter : function( ev ){ + //console.debug( 'dragenter:', this, ev ); + ev.preventDefault(); + ev.stopPropagation(); + this.$( '.history-drop-target' ).css( 'border', '2px solid black' ); + }, + /** */ + dragover : function( ev ){ + ev.preventDefault(); + ev.stopPropagation(); + }, + /** */ + dragleave : function( ev ){ + //console.debug( 'dragleave:', this, ev ); + ev.preventDefault(); + ev.stopPropagation(); + this.$( '.history-drop-target' ).css( 'border', '1px dashed black' ); + }, + /** */ + drop : function( ev ){ + //console.warn( 'dataTransfer:', ev.dataTransfer.getData( 'text' ) ); + //console.warn( 'dataTransfer:', ev.originalEvent.dataTransfer.getData( 'text' ) ); + ev.preventDefault(); + //ev.stopPropagation(); + ev.dataTransfer.dropEffect = 'move'; + + //console.debug( 'ev.dataTransfer:', ev.dataTransfer ); + + var panel = this, + data = ev.dataTransfer.getData( "text" ); + try { + data = JSON.parse( data ); + + } catch( err ){ + this.warn( 'error parsing JSON from drop:', data ); + } + this.trigger( 'droptarget:drop', ev, data, panel ); + return false; + }, + + /** */ + dataDropped : function( data ){ + var panel = this; + // HDA: dropping will copy it to the history + if( _.isObject( data ) && data.model_class === 'HistoryDatasetAssociation' && data.id ){ + return panel.model.contents.copy( data.id ); + } + return jQuery.when(); + }, + // ........................................................................ misc /** Return a string rep of the history */ toString : function(){ diff -r 605de23ca239879a34426885feddd2d3a459ca26 -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee client/galaxy/scripts/mvc/history/history-panel.js --- a/client/galaxy/scripts/mvc/history/history-panel.js +++ b/client/galaxy/scripts/mvc/history/history-panel.js @@ -254,6 +254,10 @@ _setUpWebStorage : function( initiallyExpanded, show_deleted, show_hidden ){ //if( !this.model ){ return this; } //this.log( '_setUpWebStorage', initiallyExpanded, show_deleted, show_hidden ); + if( this.storage ){ + this.stopListening( this.storage ); + } + this.storage = new HistoryPrefs({ id: HistoryPrefs.historyStorageKey( this.model.get( 'id' ) ) }); @@ -276,6 +280,18 @@ this.trigger( 'new-storage', this.storage, this ); this.log( this + ' (init\'d) storage:', this.storage.get() ); + + this.listenTo( this.storage, { + 'change:show_deleted' : function( view, newVal ){ + this.showDeleted = newVal; + }, + 'change:show_hidden' : function( view, newVal ){ + this.showHidden = newVal; + } + }, this ); + this.showDeleted = ( show_deleted !== undefined )? show_deleted : this.storage.get( 'show_deleted' ); + this.showHidden = ( show_hidden !== undefined )? show_hidden : this.storage.get( 'show_hidden' ); + return this; }, @@ -317,8 +333,8 @@ _filterItem : function( model ){ var panel = this; return ( _super.prototype._filterItem.call( panel, model ) - && ( !model.hidden() || panel.storage.get( 'show_hidden' ) ) - && ( !model.isDeletedOrPurged() || panel.storage.get( 'show_deleted' ) ) ); + && ( !model.hidden() || panel.showHidden ) + && ( !model.isDeletedOrPurged() || panel.showDeleted ) ); }, /** in this override, add a linktarget, and expand if id is in web storage */ @@ -372,12 +388,17 @@ * (2) re-rendering the history * @returns {Boolean} new show_deleted setting */ - toggleShowDeleted : function( show ){ - show = ( show !== undefined )?( show ):( !this.storage.get( 'show_deleted' ) ); - this.storage.set( 'show_deleted', show ); + toggleShowDeleted : function( show, store ){ + show = ( show !== undefined )?( show ):( !this.showDeleted ); + store = ( store !== undefined )?( store ):( true ); + this.showDeleted = show; + if( store ){ + this.storage.set( 'show_deleted', show ); + } + this.trigger( 'show-hidden', show ); //TODO:?? to events on storage('change:show_deleted') this.renderItems(); - return this.storage.get( 'show_deleted' ); + return this.showDeleted; }, /** Handle the user toggling the deleted visibility by: @@ -385,12 +406,17 @@ * (2) re-rendering the history * @returns {Boolean} new show_hidden setting */ - toggleShowHidden : function( show ){ - show = ( show !== undefined )?( show ):( !this.storage.get( 'show_hidden' ) ); - this.storage.set( 'show_hidden', show ); - //TODO:?? to events on storage('change:show_hidden') + toggleShowHidden : function( show, store ){ + show = ( show !== undefined )?( show ):( !this.showHidden ); + store = ( store !== undefined )?( store ):( true ); + this.showHidden = show; + if( store ){ + this.storage.set( 'show_hidden', show ); + } + this.trigger( 'show-hidden', show ); + //TODO:?? to events on storage('change:show_deleted') this.renderItems(); - return this.storage.get( 'show_hidden' ); + return this.showHidden; }, /** On the first search, if there are no details - load them, then search */ @@ -582,9 +608,7 @@ '<div class="title">', '<div class="name"><%= history.name %></div>', '</div>', - '<div class="subtitle">', - //'<%= view.collection.length %>', _l( ' items' ), - '</div>', + '<div class="subtitle"></div>', '<div class="history-size"><%= history.nice_size %></div>', '<div class="actions"></div>', This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/055016b332aa/ Changeset: 055016b332aa Branch: stable User: natefoo Date: 2015-01-13 15:36:58+00:00 Summary: Added tag release_2015.01.13 for changeset 2e8dd2949dd3 Affected #: 1 file diff -r 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee -r 055016b332aa41d559f9ec12c638734824292b54 .hgtags --- a/.hgtags +++ b/.hgtags @@ -21,3 +21,4 @@ 8150024c0e6fc5aef3033cf8aaa574896f6b5d0d latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 c437b28348a9345db8433e5b4f0e05ec8fb6c38a latest_2014.10.06 +2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee release_2015.01.13 https://bitbucket.org/galaxy/galaxy-central/commits/fd1946912be3/ Changeset: fd1946912be3 Branch: stable User: natefoo Date: 2015-01-13 15:37:28+00:00 Summary: Added tag latest_2015.01.13 for changeset 2e8dd2949dd3 Affected #: 1 file diff -r 055016b332aa41d559f9ec12c638734824292b54 -r fd1946912be3f0454e155effc10e5a2127388559 .hgtags --- a/.hgtags +++ b/.hgtags @@ -22,3 +22,4 @@ 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 c437b28348a9345db8433e5b4f0e05ec8fb6c38a latest_2014.10.06 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee release_2015.01.13 +2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee latest_2015.01.13 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Remove pointless user check preventing userA from requesting reset for userB. userA could just log out and then request a password reset for userB.
by commits-noreply＠bitbucket.org 13 Jan '15

13 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a74b0112995c/ Changeset: a74b0112995c User: dannon Date: 2015-01-13 21:36:10+00:00 Summary: Remove pointless user check preventing userA from requesting reset for userB. userA could just log out and then request a password reset for userB. Affected #: 1 file diff -r 5e6da19ffd907f39afd795e5d43b306071556bad -r a74b0112995cd2ae396d0916a6e845b7709f9002 lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1149,33 +1149,27 @@ reset_user = trans.sa_session.query( trans.app.model.User ).filter( trans.app.model.User.table.c.email == email ).first() user = trans.get_user() if reset_user: - if user and user.id != reset_user.id: - # This doesn't make any sense because all they have to do is log - # out and then try it again #TODO revisit why this exists. - message = "You may only reset your own password." + prt = trans.app.model.PasswordResetToken( reset_user ) + trans.sa_session.add( prt ) + trans.sa_session.flush() + host = trans.request.host.split( ':' )[ 0 ] + if host == 'localhost': + host = socket.getfqdn() + reset_url = url_for( controller='user', + action="change_password", + token=prt.token, qualified=True) + body = PASSWORD_RESET_TEMPLATE % ( host, reset_url, reset_url ) + frm = 'galaxy-no-reply@' + host + subject = 'Galaxy Password Reset' + try: + util.send_mail( frm, email, subject, body, trans.app.config ) + trans.sa_session.add( reset_user ) + trans.sa_session.flush() + trans.log_event( "User reset password: %s" % email ) + except Exception, e: status = 'error' - else: - prt = trans.app.model.PasswordResetToken( reset_user ) - trans.sa_session.add( prt ) - trans.sa_session.flush() - host = trans.request.host.split( ':' )[ 0 ] - if host == 'localhost': - host = socket.getfqdn() - reset_url = url_for( controller='user', - action="change_password", - token=prt.token, qualified=True) - body = PASSWORD_RESET_TEMPLATE % ( host, reset_url, reset_url ) - frm = 'galaxy-no-reply@' + host - subject = 'Galaxy Password Reset' - try: - util.send_mail( frm, email, subject, body, trans.app.config ) - trans.sa_session.add( reset_user ) - trans.sa_session.flush() - trans.log_event( "User reset password: %s" % email ) - except Exception, e: - status = 'error' - message = 'Failed to reset password: %s' % str( e ) - log.exception( 'Unable to reset password.' ) + message = 'Failed to reset password: %s' % str( e ) + log.exception( 'Unable to reset password.' ) return trans.fill_template( '/user/reset_password.mako', message=message, status=status ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Make password reset via link feature work for the toolshed too.
by commits-noreply＠bitbucket.org 13 Jan '15

13 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5e6da19ffd90/ Changeset: 5e6da19ffd90 User: dannon Date: 2015-01-13 21:27:35+00:00 Summary: Make password reset via link feature work for the toolshed too. Affected #: 5 files diff -r d2cafc77eee48d07b1958e395f6a8cf58770cf94 -r 5e6da19ffd907f39afd795e5d43b306071556bad lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1155,7 +1155,7 @@ message = "You may only reset your own password." status = 'error' else: - prt = model.PasswordResetToken( reset_user ) + prt = trans.app.model.PasswordResetToken( reset_user ) trans.sa_session.add( prt ) trans.sa_session.flush() host = trans.request.host.split( ':' )[ 0 ] diff -r d2cafc77eee48d07b1958e395f6a8cf58770cf94 -r 5e6da19ffd907f39afd795e5d43b306071556bad lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -1,7 +1,9 @@ import logging import operator import os +from datetime import datetime, timedelta from galaxy import util +from galaxy.util import unique_id from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash from galaxy.model.item_attrs import Dictifiable @@ -61,6 +63,16 @@ self.password = new_secure_hash( text_type=cleartext ) +class PasswordResetToken( object ): + def __init__( self, user, token=None): + if token: + self.token = token + else: + self.token = unique_id() + self.user = user + self.expiration_time = datetime.now() + timedelta(hours=24) + + class Group( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name' ) dict_element_visible_keys = ( 'id', 'name' ) @@ -85,7 +97,7 @@ self.description = description self.type = type self.deleted = deleted - + @property def is_repository_admin_role( self ): # A repository admin role must always be associated with a repository. The mapper returns an diff -r d2cafc77eee48d07b1958e395f6a8cf58770cf94 -r 5e6da19ffd907f39afd795e5d43b306071556bad lib/galaxy/webapps/tool_shed/model/mapping.py --- a/lib/galaxy/webapps/tool_shed/model/mapping.py +++ b/lib/galaxy/webapps/tool_shed/model/mapping.py @@ -37,6 +37,11 @@ Column( "deleted", Boolean, index=True, default=False ), Column( "purged", Boolean, index=True, default=False ) ) +PasswordResetToken.table = Table("password_reset_token", metadata, + Column( "token", String( 32 ), primary_key=True, unique=True, index=True ), + Column( "expiration_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) + Group.table = Table( "galaxy_group", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -204,6 +209,9 @@ galaxy_sessions=relation( GalaxySession, order_by=desc( GalaxySession.table.c.update_time ) ), api_keys=relation( APIKeys, backref="user", order_by=desc( APIKeys.table.c.create_time ) ) ) ) +mapper( PasswordResetToken, PasswordResetToken.table, + properties=dict( user=relation( User, backref="reset_tokens") ) ) + mapper( APIKeys, APIKeys.table, properties = {} ) diff -r d2cafc77eee48d07b1958e395f6a8cf58770cf94 -r 5e6da19ffd907f39afd795e5d43b306071556bad lib/galaxy/webapps/tool_shed/model/migrate/versions/0024_password_reset.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0024_password_reset.py @@ -0,0 +1,43 @@ +""" +Migration script for the password reset table +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +from galaxy.model.custom_types import * + +import datetime +now = datetime.datetime.utcnow + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData() + +PasswordResetToken_table = Table("password_reset_token", metadata, + Column( "token", String( 32 ), primary_key=True, unique=True, index=True ), + Column( "expiration_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )) + + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + try: + PasswordResetToken_table.create() + except Exception as e: + print str(e) + log.exception("Creating %s table failed: %s" % (PasswordResetToken_table.name, str( e ) ) ) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + try: + PasswordResetToken_table.drop() + except Exception as e: + print str(e) + log.exception("Dropping %s table failed: %s" % (PasswordResetToken_table.name, str( e ) ) ) diff -r d2cafc77eee48d07b1958e395f6a8cf58770cf94 -r 5e6da19ffd907f39afd795e5d43b306071556bad templates/user/index.mako --- a/templates/user/index.mako +++ b/templates/user/index.mako @@ -18,6 +18,7 @@ <li><a href="${h.url_for( controller='user', action='logout', logout_all=True )}" target="_top">${_('Logout')}</a> ${_('of all user sessions')}</li> %else: <li><a href="${h.url_for( controller='user', action='manage_user_info', cntrller=cntrller )}">${_('Manage your information')}</a></li> + <li><a href="${h.url_for( controller='user', action='change_password' )}">${_('Change your password')}</a></li><li><a href="${h.url_for( controller='user', action='api_keys', cntrller=cntrller )}">${_('Manage your API keys')}</a></li><li><a href="${h.url_for( controller='repository', action='manage_email_alerts', cntrller=cntrller )}">${_('Manage your email alerts')}</a></li><li><a href="${h.url_for( controller='user', action='logout', logout_all=True )}" target="_top">${_('Logout')}</a> ${_('of all user sessions')}</li> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 8 new changesets
by commits-noreply＠bitbucket.org 13 Jan '15

13 Jan '15

8 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/109372878f62/ Changeset: 109372878f62 User: dannon Date: 2015-01-13 19:17:33+00:00 Summary: Initial password reset token feature. Affected #: 9 files diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -354,10 +354,6 @@ # If no message specified the warning box will not be shown. #registration_warning_message = Please register only one account - we provide this service free of charge and have limited computational resources. Multi-accounts are tracked and will be subjected to account termination and data deletion. -# When users opt to reset passwords, new ones are created, this option -# specifies the length of these passwords. -#reset_password_length = 15 - # -- Account activation diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -161,7 +161,6 @@ self.job_walltime_delta = timedelta( 0, s, 0, 0, m, h ) self.admin_users = kwargs.get( "admin_users", "" ) self.admin_users_list = [u.strip() for u in self.admin_users.split(',') if u] - self.reset_password_length = int( kwargs.get('reset_password_length', '15') ) self.mailing_join_addr = kwargs.get('mailing_join_addr', 'galaxy-announce-join(a)bx.psu.edu') self.error_email_to = kwargs.get( 'error_email_to', None ) self.activation_email = kwargs.get( 'activation_email', None ) diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -18,6 +18,7 @@ import socket import time import numbers +from datetime import datetime, timedelta from uuid import UUID, uuid4 from string import Template from itertools import ifilter @@ -31,7 +32,7 @@ import galaxy.model.orm.now from galaxy.security import get_permitted_actions from galaxy.util import is_multi_byte, nice_size, Params, restore_text, send_mail -from galaxy.util import ready_name_for_url +from galaxy.util import ready_name_for_url, unique_id from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash from galaxy.util.directory_hash import directory_hash_id @@ -261,6 +262,14 @@ environment = User.user_template_environment( user ) return Template( in_string ).safe_substitute( environment ) +class PasswordResetToken( object ): + def __init__( self, user, token=None): + if token: + self.token = token + else: + self.token = unique_id() + self.user = user + self.expiration_time = datetime.now() + timedelta(hours=24) class BaseJobMetric( object ): diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -68,6 +68,11 @@ Column( "provider", TrimmedString( 255 ) ), ) +model.PasswordResetToken.table = Table("password_reset_token", metadata, + Column( "token", String( 32 ), primary_key=True, unique=True, index=True ), + Column( "expiration_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) + model.History.table = Table( "history", metadata, Column( "id", Integer, primary_key=True), Column( "create_time", DateTime, default=now ), @@ -1564,6 +1569,10 @@ api_keys=relation( model.APIKeys, backref="user", order_by=desc( model.APIKeys.table.c.create_time ) ), ) ) +mapper( model.PasswordResetToken, model.PasswordResetToken.table, + properties=dict( user=relation( model.User, backref="reset_tokens") ) ) + + # Set up proxy so that this syntax is possible: # <user_obj>.preferences[pref_name] = pref_value model.User.preferences = association_proxy('_preferences', 'value', creator=model.UserPreference) diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c lib/galaxy/model/migrate/versions/0126_password_reset.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0126_password_reset.py @@ -0,0 +1,43 @@ +""" +Migration script for the password reset table +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +from galaxy.model.custom_types import * + +import datetime +now = datetime.datetime.utcnow + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData() + +PasswordResetToken_table = Table("password_reset_token", metadata, + Column( "token", String( 32 ), primary_key=True, unique=True, index=True ), + Column( "expiration_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )) + + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + try: + PasswordResetToken_table.create() + except Exception as e: + print str(e) + log.exception("Creating %s table failed: %s" % (PasswordResetToken_table.name, str( e ) ) ) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + try: + PasswordResetToken_table.drop() + except Exception as e: + print str(e) + log.exception("Dropping %s table failed: %s" % (PasswordResetToken_table.name, str( e ) ) ) diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -7,7 +7,6 @@ import os import random import socket -import string import urllib from datetime import datetime, timedelta @@ -41,6 +40,13 @@ <p/> """ +PASSWORD_RESET_TEMPLATE = """ +To reset your Galaxy password for the instance at %s, use the following link: + + <a href="%s">%s</a> + +If you did not make this request, no action is necessary on your part, though +you may want to notify an administrator.""" class UserOpenIDGrid( grids.Grid ): use_panels = False @@ -1031,38 +1037,6 @@ trans.sa_session.add( user ) trans.sa_session.flush() message = 'The login information has been updated with the changes.' - elif user and params.get( 'change_password_button', False ): - # Editing password. Do not sanitize passwords, so get from kwd - # and not params (which were sanitized). - password = kwd.get( 'password', '' ) - confirm = kwd.get( 'confirm', '' ) - ok = True - if not is_admin: - # If the current user is changing their own password, validate their current password - current = kwd.get( 'current', '' ) - if not trans.user.check_password( current ): - message = 'Invalid current password' - status = 'error' - ok = False - if ok: - # Validate the new password - message = validate_password( trans, password, confirm ) - if message: - status = 'error' - else: - # Save new password - user.set_password_cleartext( password ) - # Invalidate all other sessions - for other_galaxy_session in trans.sa_session.query( trans.app.model.GalaxySession ) \ - .filter( and_( trans.app.model.GalaxySession.table.c.user_id == trans.user.id, - trans.app.model.GalaxySession.table.c.is_valid is True, - trans.app.model.GalaxySession.table.c.id != trans.galaxy_session.id ) ): - other_galaxy_session.is_valid = False - trans.sa_session.add( other_galaxy_session ) - trans.sa_session.add( user ) - trans.sa_session.flush() - trans.log_event( "User change password" ) - message = 'The password has been changed and any other existing Galaxy sessions have been logged out (but jobs in histories in those sessions will not be interrupted).' elif user and params.get( 'edit_user_info_button', False ): # Edit user information - webapp MUST BE 'galaxy' user_type_fd_id = params.get( 'user_type_fd_id', 'none' ) @@ -1105,38 +1079,99 @@ **kwd ) ) @web.expose - def reset_password( self, trans, email=None, **kwd ): + def change_password( self, trans, token=None, **kwd): + """ + Provides a form with which one can change their password. If token is + provided, don't require current password. + """ + status = None + message = None + user = None + if kwd.get( 'change_password_button', False ): + password = kwd.get( 'password', '' ) + confirm = kwd.get( 'confirm', '' ) + current = kwd.get( 'current', '' ) + token_result = None + if token: + # If a token was supplied, validate and set user + token_result = trans.sa_session.query( trans.app.model.PasswordResetToken ).get(token) + if token_result and token_result.expiration_time > datetime.now(): + user = token_result.user + else: + return trans.show_error_message("Invalid or expired password reset token, please request a new one.") + else: + # The user is changing their own password, validate their current password + if trans.user.check_password( current ): + user = trans.user + else: + message = 'Invalid current password' + status = 'error' + if user: + # Validate the new password + message = validate_password( trans, password, confirm ) + if message: + status = 'error' + else: + # Save new password + user.set_password_cleartext( password ) + # if we used a token, invalidate it and log the user in. + if token_result: + trans.handle_user_login(token_result.user) + token_result.expiration_time = datetime.now() + trans.sa_session.add(token_result) + # Invalidate all other sessions + for other_galaxy_session in trans.sa_session.query( trans.app.model.GalaxySession ) \ + .filter( and_( trans.app.model.GalaxySession.table.c.user_id == user.id, + trans.app.model.GalaxySession.table.c.is_valid is True, + trans.app.model.GalaxySession.table.c.id != trans.galaxy_session.id ) ): + other_galaxy_session.is_valid = False + trans.sa_session.add( other_galaxy_session ) + trans.sa_session.add( user ) + trans.sa_session.flush() + trans.log_event( "User change password" ) + return trans.show_ok_message('The password has been changed and any other existing Galaxy sessions have been logged out (but jobs in histories in those sessions will not be interrupted).') + return trans.fill_template( '/user/change_password.mako', + token=token, + status=status, + message=message ) + + + @web.expose + def reset_password( self, trans, email=None, token=None, **kwd ): """Reset the user's password. Send an email with the new password.""" if trans.app.config.smtp_server is None: - return trans.show_error_message( "Mail is not configured for this Galaxy instance. Please contact your local Galaxy administrator." ) - message = '' - status = 'done' + return trans.show_error_message( "Mail is not configured for this" + "Galaxy instance and password reset information cannot be sent." + "Please contact your local Galaxy administrator." ) + message = None + status = None if kwd.get( 'reset_password_button', False ): reset_user = trans.sa_session.query( trans.app.model.User ).filter( trans.app.model.User.table.c.email == email ).first() user = trans.get_user() if reset_user: if user and user.id != reset_user.id: + # This doesn't make any sense because all they have to do is log + # out and then try it again #TODO revisit why this exists. message = "You may only reset your own password." status = 'error' else: - chars = string.letters + string.digits - new_pass = "" + prt = model.PasswordResetToken(reset_user) + trans.sa_session.add(prt) + trans.sa_session.flush() reset_password_length = getattr( trans.app.config, "reset_password_length", 15 ) - for i in range( reset_password_length ): - new_pass = new_pass + random.choice( chars ) host = trans.request.host.split( ':' )[ 0 ] if host == 'localhost': host = socket.getfqdn() - body = 'Your password on %s has been reset to:\n\n %s\n' % ( host, new_pass ) + reset_url = url_for( controller='user', action="change_password", token=prt.token, qualified=True) + body = PASSWORD_RESET_TEMPLATE % ( host, reset_url, reset_url ) frm = 'galaxy-no-reply@' + host subject = 'Galaxy Password Reset' try: util.send_mail( frm, email, subject, body, trans.app.config ) - reset_user.set_password_cleartext( new_pass ) trans.sa_session.add( reset_user ) trans.sa_session.flush() trans.log_event( "User reset password: %s" % email ) - message = "Password has been reset and emailed to: %s. <a href='%s'>Click here</a> to return to the login form." % ( escape( email ), web.url_for( controller='user', action='login', noredirect='true' ) ) + message = "An email has been sent to %s. Please refer to that email for more instructions." % ( escape( email ) ) except Exception, e: status = 'error' message = 'Failed to reset password: %s' % str( e ) diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c templates/user/change_password.mako --- /dev/null +++ b/templates/user/change_password.mako @@ -0,0 +1,31 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, status )} +%endif + +<div class="toolForm"> + <form name="change_password" id="change_password" action="${h.url_for( controller='user', action='change_password' )}" method="post" > + <div class="toolFormTitle">Change Password</div> + %if token: + <input type="hidden" name="token" value="${token|h}"/> + %elif not is_admin: + <div class="form-row"> + <label>Current password:</label> + <input type="password" name="current" value="" size="40"/> + </div> + %endif + <div class="form-row"> + <label>New password:</label> + <input type="password" name="password" value="" size="40"/> + </div> + <div class="form-row"> + <label>Confirm:</label> + <input type="password" name="confirm" value="" size="40"/> + </div> + <div class="form-row"> + <input type="submit" name="change_password_button" value="Save"/> + </div> + </form> +</div> diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c templates/user/index.mako --- a/templates/user/index.mako +++ b/templates/user/index.mako @@ -6,7 +6,8 @@ <ul> %if t.webapp.name == 'galaxy': %if not trans.app.config.use_remote_user: - <li><a href="${h.url_for( controller='user', action='manage_user_info', cntrller=cntrller )}">${_('Manage your information')}</a> (email, password, etc.)</li> + <li><a href="${h.url_for( controller='user', action='manage_user_info', cntrller=cntrller )}">${_('Manage your information')}</a> (email, address, etc.)</li> + <li><a href="${h.url_for( controller='user', action='change_password', cntrller=cntrller )}">${_('Change your password')}</a></li> %endif <li><a href="${h.url_for( controller='user', action='set_default_permissions', cntrller=cntrller )}">${_('Change default permissions')}</a> for new histories</li><li><a href="${h.url_for( controller='user', action='api_keys', cntrller=cntrller )}">${_('Manage your API keys')}</a></li> diff -r a2308bdc93b897af974766b190abe019ade49e9a -r 109372878f62317b74b57b47bb37d44b486ccf0c templates/user/info.mako --- a/templates/user/info.mako +++ b/templates/user/info.mako @@ -127,27 +127,4 @@ </div></form></div> - <p></p> - <div class="toolForm"> - <form name="change_password" id="change_password" action="${h.url_for( controller='user', action='edit_info', cntrller=cntrller, user_id=trans.security.encode_id( user.id ) )}" method="post" > - <div class="toolFormTitle">Change Password</div> - %if not is_admin: - <div class="form-row"> - <label>Current password:</label> - <input type="password" name="current" value="" size="40"/> - </div> - %endif - <div class="form-row"> - <label>New password:</label> - <input type="password" name="password" value="" size="40"/> - </div> - <div class="form-row"> - <label>Confirm:</label> - <input type="password" name="confirm" value="" size="40"/> - </div> - <div class="form-row"> - <input type="submit" name="change_password_button" value="Save"/> - </div> - </form> - </div></%def> https://bitbucket.org/galaxy/galaxy-central/commits/d0de709cc3e1/ Changeset: d0de709cc3e1 User: dannon Date: 2015-01-13 19:21:08+00:00 Summary: pep8, import dedupe in user controller. Affected #: 1 file diff -r 109372878f62317b74b57b47bb37d44b486ccf0c -r d0de709cc3e16ca6e0b9e28ee91ea124467081a4 lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -28,12 +28,10 @@ UsesFormDefinitionsMixin) from galaxy.web.form_builder import build_select_field, CheckboxField from galaxy.web.framework.helpers import escape, grids, time_ago -from galaxy.web.framework.helpers import time_ago, grids -from markupsafe import escape log = logging.getLogger( __name__ ) -require_login_template = """ +REQUIRE_LOGIN_TEMPLATE = """ <p> This %s has been configured such that only users who are logged in may use it.%s </p> @@ -48,6 +46,7 @@ If you did not make this request, no action is necessary on your part, though you may want to notify an administrator.""" + class UserOpenIDGrid( grids.Grid ): use_panels = False title = "OpenIDs linked to your account" @@ -483,14 +482,14 @@ create_account_str = " If you don't already have an account, <a href='%s'>you may create one</a>." % \ web.url_for( controller='user', action='create', cntrller='user' ) if trans.webapp.name == 'galaxy': - header = require_login_template % ( "Galaxy instance", create_account_str ) + header = REQUIRE_LOGIN_TEMPLATE % ( "Galaxy instance", create_account_str ) else: - header = require_login_template % ( "Galaxy tool shed", create_account_str ) + header = REQUIRE_LOGIN_TEMPLATE % ( "Galaxy tool shed", create_account_str ) else: if trans.webapp.name == 'galaxy': - header = require_login_template % ( "Galaxy instance", "" ) + header = REQUIRE_LOGIN_TEMPLATE % ( "Galaxy instance", "" ) else: - header = require_login_template % ( "Galaxy tool shed", "" ) + header = REQUIRE_LOGIN_TEMPLATE % ( "Galaxy tool shed", "" ) return trans.fill_template( '/user/login.mako', email=email, header=header, @@ -1135,14 +1134,13 @@ status=status, message=message ) - @web.expose def reset_password( self, trans, email=None, token=None, **kwd ): """Reset the user's password. Send an email with the new password.""" if trans.app.config.smtp_server is None: - return trans.show_error_message( "Mail is not configured for this" - "Galaxy instance and password reset information cannot be sent." - "Please contact your local Galaxy administrator." ) + return trans.show_error_message( "Mail is not configured for this Galaxy instance " + "and password reset information cannot be sent." + "Please contact your local Galaxy administrator." ) message = None status = None if kwd.get( 'reset_password_button', False ): https://bitbucket.org/galaxy/galaxy-central/commits/9fa4eee8e77c/ Changeset: 9fa4eee8e77c User: dannon Date: 2015-01-13 19:22:03+00:00 Summary: Remove unused old password_reset_length for good. Affected #: 1 file diff -r d0de709cc3e16ca6e0b9e28ee91ea124467081a4 -r 9fa4eee8e77c2605b123ca9f686dd1970a1bfd2a lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1156,7 +1156,6 @@ prt = model.PasswordResetToken(reset_user) trans.sa_session.add(prt) trans.sa_session.flush() - reset_password_length = getattr( trans.app.config, "reset_password_length", 15 ) host = trans.request.host.split( ':' )[ 0 ] if host == 'localhost': host = socket.getfqdn() https://bitbucket.org/galaxy/galaxy-central/commits/80361f1c960f/ Changeset: 80361f1c960f User: dannon Date: 2015-01-13 19:24:15+00:00 Summary: Style tweaks in user_info.mako Affected #: 1 file diff -r 9fa4eee8e77c2605b123ca9f686dd1970a1bfd2a -r 80361f1c960fdbe48e83549f92c001d36d72a78d templates/user/info.mako --- a/templates/user/info.mako +++ b/templates/user/info.mako @@ -7,7 +7,7 @@ <script type="text/javascript"> $(document).ready(function() { - function validateString(test_string, type) { + function validateString(test_string, type) { var mail_re = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; var username_re = /^[a-z0-9\-]{4,255}$/; if (type === 'email') { @@ -15,7 +15,7 @@ } else if (type === 'username'){ return username_re.test(test_string); } - } + } function renderError(message) { $(".donemessage").hide(); @@ -26,8 +26,9 @@ div.className = "errormessage"; div.innerHTML = message; document.body.insertBefore( div, document.body.firstChild ); - } } + } + function renderDone(message) { $(".errormessage").hide(); if ($(".donemessage").length === 1) { @@ -37,8 +38,8 @@ div.className = "donemessage"; div.innerHTML = message; document.body.insertBefore( div, document.body.firstChild ); - } } + } original_email = $( '#email_input' ).val(); original_username = $( '#name_input' ).val(); @@ -55,7 +56,6 @@ var hidden_input = '<input type="hidden" id="login_info_button" name="login_info_button" value="Submit"/>'; $( '#send' ).attr( 'disabled', 'disabled' ); $( "#email_input" ).before( hidden_input ); - if ( original_email !== email ){ if ( email.length > 255 ){ renderError( error_text_email_long ); validForm = false; } else if ( !validateString( email, "email" ) ){ renderError( error_text_email ); validForm = false; } @@ -66,7 +66,7 @@ if ( nothing_changed ){ renderDone( "Nothing has changed." ); } - if ( !validForm || nothing_changed ) { + if ( !validForm || nothing_changed ) { e.preventDefault(); // reactivate the button if the form wasn't submitted $( '#send' ).removeAttr( 'disabled' ); https://bitbucket.org/galaxy/galaxy-central/commits/b64ae555ad12/ Changeset: b64ae555ad12 User: dannon Date: 2015-01-13 19:33:56+00:00 Summary: More minor style tweaks in user controller. Affected #: 1 file diff -r 80361f1c960fdbe48e83549f92c001d36d72a78d -r b64ae555ad12fa3fef9a96954b33f8f23b723893 lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1153,13 +1153,15 @@ message = "You may only reset your own password." status = 'error' else: - prt = model.PasswordResetToken(reset_user) - trans.sa_session.add(prt) + prt = model.PasswordResetToken( reset_user ) + trans.sa_session.add( prt ) trans.sa_session.flush() host = trans.request.host.split( ':' )[ 0 ] if host == 'localhost': host = socket.getfqdn() - reset_url = url_for( controller='user', action="change_password", token=prt.token, qualified=True) + reset_url = url_for( controller='user', + action="change_password", + token=prt.token, qualified=True) body = PASSWORD_RESET_TEMPLATE % ( host, reset_url, reset_url ) frm = 'galaxy-no-reply@' + host subject = 'Galaxy Password Reset' https://bitbucket.org/galaxy/galaxy-central/commits/27fec2663c6c/ Changeset: 27fec2663c6c User: dannon Date: 2015-01-13 19:58:35+00:00 Summary: Default to a response that doesn't leak user information (whether or not the user exists, when resetting password. Affected #: 1 file diff -r b64ae555ad12fa3fef9a96954b33f8f23b723893 -r 27fec2663c6c9305926f6bc34ba074042dd467cb lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1142,8 +1142,10 @@ "and password reset information cannot be sent." "Please contact your local Galaxy administrator." ) message = None - status = None + status = 'done' if kwd.get( 'reset_password_button', False ): + # Default to a non-userinfo-leaking response message + message = "Your reset request for %s has been received. Please check your email account for more instructions. If you do not receive an email shortly, please contact an administrator." % ( escape( email ) ) reset_user = trans.sa_session.query( trans.app.model.User ).filter( trans.app.model.User.table.c.email == email ).first() user = trans.get_user() if reset_user: @@ -1170,16 +1172,10 @@ trans.sa_session.add( reset_user ) trans.sa_session.flush() trans.log_event( "User reset password: %s" % email ) - message = "An email has been sent to %s. Please refer to that email for more instructions." % ( escape( email ) ) except Exception, e: status = 'error' message = 'Failed to reset password: %s' % str( e ) log.exception( 'Unable to reset password.' ) - elif email is not None: - message = "The specified user does not exist." - status = 'error' - elif email is None: - email = "" return trans.fill_template( '/user/reset_password.mako', message=message, status=status ) https://bitbucket.org/galaxy/galaxy-central/commits/5222b149d9dc/ Changeset: 5222b149d9dc User: dannon Date: 2015-01-13 20:08:55+00:00 Summary: Change password does not need 'cntrller'. Affected #: 1 file diff -r 27fec2663c6c9305926f6bc34ba074042dd467cb -r 5222b149d9dcdd84fe593e64e3b48c3c7a0bc395 templates/user/index.mako --- a/templates/user/index.mako +++ b/templates/user/index.mako @@ -7,7 +7,7 @@ %if t.webapp.name == 'galaxy': %if not trans.app.config.use_remote_user: <li><a href="${h.url_for( controller='user', action='manage_user_info', cntrller=cntrller )}">${_('Manage your information')}</a> (email, address, etc.)</li> - <li><a href="${h.url_for( controller='user', action='change_password', cntrller=cntrller )}">${_('Change your password')}</a></li> + <li><a href="${h.url_for( controller='user', action='change_password' )}">${_('Change your password')}</a></li> %endif <li><a href="${h.url_for( controller='user', action='set_default_permissions', cntrller=cntrller )}">${_('Change default permissions')}</a> for new histories</li><li><a href="${h.url_for( controller='user', action='api_keys', cntrller=cntrller )}">${_('Manage your API keys')}</a></li> https://bitbucket.org/galaxy/galaxy-central/commits/d2cafc77eee4/ Changeset: d2cafc77eee4 User: dannon Date: 2015-01-13 20:31:11+00:00 Summary: Correct spacing in mail configuration message, was missing one between sentences. Affected #: 1 file diff -r 5222b149d9dcdd84fe593e64e3b48c3c7a0bc395 -r d2cafc77eee48d07b1958e395f6a8cf58770cf94 lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1139,7 +1139,7 @@ """Reset the user's password. Send an email with the new password.""" if trans.app.config.smtp_server is None: return trans.show_error_message( "Mail is not configured for this Galaxy instance " - "and password reset information cannot be sent." + "and password reset information cannot be sent. " "Please contact your local Galaxy administrator." ) message = None status = 'done' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0