February 2014 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: guerler: Upload: Hide former upload tool
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/26e37f6af3a8/ Changeset: 26e37f6af3a8 User: guerler Date: 2014-02-18 20:57:08 Summary: Upload: Hide former upload tool Affected #: 3 files diff -r c755326c2018efa988daebe96c355a32b6482018 -r 26e37f6af3a8a090d9d6e923fa3e90acb0e64572 static/scripts/mvc/upload/upload-view.js --- a/static/scripts/mvc/upload/upload-view.js +++ b/static/scripts/mvc/upload/upload-view.js @@ -183,7 +183,7 @@ buttons : { 'Choose local file' : function() {self.uploadbox.select()}, 'Choose FTP file' : function() {self._eventFtp()}, - 'Create new file' : function() {self._eventCreate()}, + 'Paste/Fetch data' : function() {self._eventCreate()}, 'Start' : function() {self._eventStart()}, 'Pause' : function() {self._eventStop()}, 'Reset' : function() {self._eventReset()}, @@ -577,11 +577,11 @@ { this.modal.enableButton('Choose local file'); this.modal.enableButton('Choose FTP file'); - this.modal.enableButton('Create new file'); + this.modal.enableButton('Paste/Fetch data'); } else { this.modal.disableButton('Choose local file'); this.modal.disableButton('Choose FTP file'); - this.modal.disableButton('Create new file'); + this.modal.disableButton('Paste/Fetch data'); } // ftp button diff -r c755326c2018efa988daebe96c355a32b6482018 -r 26e37f6af3a8a090d9d6e923fa3e90acb0e64572 static/scripts/packed/mvc/upload/upload-view.js --- a/static/scripts/packed/mvc/upload/upload-view.js +++ b/static/scripts/packed/mvc/upload/upload-view.js @@ -1,1 +1,1 @@ -define(["galaxy.modal","utils/utils","mvc/upload/upload-button","mvc/upload/upload-model","mvc/upload/upload-row","mvc/upload/upload-ftp","mvc/ui/ui-popover","mvc/ui","utils/uploadbox"],function(a,f,e,c,b,g,d){return Backbone.View.extend({options:{nginx_upload_path:""},modal:null,ui_button:null,uploadbox:null,current_history:null,upload_size:0,list_extensions:[],list_genomes:[],auto:{id:"auto",text:"Auto-detect",description:"This system will try to detect the file type automatically. If your file is not detected properly as one of the known formats, it most likely means that it has some format problems (e.g., different number of columns on different rows). You can still coerce the system to set your data to the format you think it should be. You can also upload compressed files, which will automatically be decompressed."},collection:new c.Collection(),ftp:null,counter:{announce:0,success:0,error:0,running:0,reset:function(){this.announce=this.success=this.error=this.running=0}},initialize:function(i){var h=this;if(i){this.options=_.defaults(i,this.options)}if(!Galaxy.currHistoryPanel||!Galaxy.currHistoryPanel.model){window.setTimeout(function(){h.initialize()},500);return}this.ui_button=new e.Model({icon:"fa-upload",tooltip:"Download from URL or upload files from disk",label:"Load Data",onclick:function(j){if(j){h._eventShow(j)}},onunload:function(){if(h.counter.running>0){return"Several uploads are still processing."}}});$("#left .unified-panel-header-inner").append((new e.View(this.ui_button)).$el);var h=this;f.jsonFromUrl(galaxy_config.root+"api/datatypes?extension_only=False",function(j){for(key in j){h.list_extensions.push({id:j[key].extension,text:j[key].extension,description:j[key].description,description_url:j[key].description_url})}h.list_extensions.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0});if(!h.options.datatypes_disable_auto){h.list_extensions.unshift(h.auto)}});f.jsonFromUrl(galaxy_config.root+"api/genomes",function(j){for(key in j){h.list_genomes.push({id:j[key][1],text:j[key][0]})}h.list_genomes.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0})});this.collection.on("remove",function(j){h._eventRemove(j)});this.collection.on("change:genome",function(k){var j=k.get("genome");h.collection.each(function(l){if(l.get("status")=="init"&&l.get("genome")=="?"){l.set("genome",j)}})})},_eventShow:function(j){j.preventDefault();if(!this.modal){var h=this;this.modal=new a.GalaxyModal({title:"Download data directly from web or upload files from your disk",body:this._template("upload-box","upload-info"),buttons:{"Choose local file":function(){h.uploadbox.select()},"Choose FTP file":function(){h._eventFtp()},"Create new file":function(){h._eventCreate()},Start:function(){h._eventStart()},Pause:function(){h._eventStop()},Reset:function(){h._eventReset()},Close:function(){h.modal.hide()},},height:"400",width:"900",closing_events:true});this.setElement("#upload-box");var h=this;this.uploadbox=this.$el.uploadbox({announce:function(k,l,m){h._eventAnnounce(k,l,m)},initialize:function(k,l,m){return h._eventInitialize(k,l,m)},progress:function(k,l,m){h._eventProgress(k,l,m)},success:function(k,l,m){h._eventSuccess(k,l,m)},error:function(k,l,m){h._eventError(k,l,m)},complete:function(){h._eventComplete()}});var i=this.modal.getButton("Choose FTP file");this.ftp=new d.View({title:"FTP files",container:i})}this.modal.show();this._updateUser();this._updateScreen()},_eventRemove:function(i){var h=i.get("status");if(h=="success"){this.counter.success--}else{if(h=="error"){this.counter.error--}else{this.counter.announce--}}this._updateScreen();this.uploadbox.remove(i.id)},_eventAnnounce:function(h,i,k){this.counter.announce++;this._updateScreen();var j=new b(this,{id:h,file_name:i.name,file_size:i.size,file_mode:i.mode,file_path:i.path});this.collection.add(j.model);$(this.el).find("tbody:first").append(j.$el);j.render()},_eventInitialize:function(m,j,s){var k=this.collection.get(m);k.set("status","running");var o=k.get("file_name");var n=k.get("file_path");var h=k.get("file_mode");var p=k.get("extension");var r=k.get("genome");var q=k.get("url_paste");var l=k.get("space_to_tabs");var i=k.get("to_posix_lines");if(!q&&!(j.size>0)){return null}this.uploadbox.configure({url:this.options.nginx_upload_path});if(h=="local"){this.uploadbox.configure({paramname:"files_0|file_data"})}else{this.uploadbox.configure({paramname:null})}tool_input={};if(h=="new"){tool_input["files_0|url_paste"]=q}if(h=="ftp"){tool_input["files_0|ftp_files"]=n}tool_input.dbkey=r;tool_input.file_type=p;tool_input["files_0|type"]="upload_dataset";tool_input.space_to_tabs=l;tool_input.to_posix_lines=i;data={};data.history_id=this.current_history;data.tool_id="upload1";data.inputs=JSON.stringify(tool_input);return data},_eventProgress:function(i,j,h){var k=this.collection.get(i);k.set("percentage",h);this.ui_button.set("percentage",this._upload_percentage(h,j.size))},_eventSuccess:function(i,j,l){var k=this.collection.get(i);k.set("percentage",100);k.set("status","success");var h=k.get("file_size");this.ui_button.set("percentage",this._upload_percentage(100,h));this.upload_completed+=h*100;this.counter.announce--;this.counter.success++;this._updateScreen();Galaxy.currHistoryPanel.refreshHdas()},_eventError:function(h,i,k){var j=this.collection.get(h);j.set("percentage",100);j.set("status","error");j.set("info",k);this.ui_button.set("percentage",this._upload_percentage(100,i.size));this.ui_button.set("status","danger");this.upload_completed+=i.size*100;this.counter.announce--;this.counter.error++;this._updateScreen()},_eventComplete:function(){this.collection.each(function(h){if(h.get("status")=="queued"){h.set("status","init")}});this.counter.running=0;this._updateScreen()},_eventFtp:function(){if(!this.ftp.visible){this.ftp.empty();this.ftp.append((new g(this)).$el);this.ftp.show()}else{this.ftp.hide()}},_eventCreate:function(){this.uploadbox.add([{name:"New File",size:0,mode:"new"}])},_eventStart:function(){if(this.counter.announce==0||this.counter.running>0){return}var h=this;this.upload_size=0;this.upload_completed=0;this.collection.each(function(i){if(i.get("status")=="init"){i.set("status","queued");h.upload_size+=i.get("file_size")}});this.ui_button.set("percentage",0);this.ui_button.set("status","success");this.counter.running=this.counter.announce;this._updateScreen();this.uploadbox.start()},_eventStop:function(){if(this.counter.running==0){return}this.ui_button.set("status","info");this.uploadbox.stop();$("#upload-info").html("Queue will pause after completing the current file...")},_eventReset:function(){if(this.counter.running==0){this.collection.reset();this.counter.reset();this._updateScreen();this.uploadbox.reset();this.ui_button.set("percentage",0)}},_updateUser:function(){this.current_user=Galaxy.currUser.get("id");this.current_history=null;if(this.current_user){this.current_history=Galaxy.currHistoryPanel.model.get("id")}},_updateScreen:function(){if(this.counter.announce==0){if(this.uploadbox.compatible()){message="You can Drag & Drop files into this box."}else{message="Unfortunately, your browser does not support multiple file uploads or drag&drop.<br>Some supported browsers are: Firefox 4+, Chrome 7+, IE 10+, Opera 12+ or Safari 6+."}}else{if(this.counter.running==0){message="You added "+this.counter.announce+" file(s) to the queue. Add more files or click 'Start' to proceed."}else{message="Please wait..."+this.counter.announce+" out of "+this.counter.running+" remaining."}}$("#upload-info").html(message);if(this.counter.running==0&&this.counter.announce+this.counter.success+this.counter.error>0){this.modal.enableButton("Reset")}else{this.modal.disableButton("Reset")}if(this.counter.running==0&&this.counter.announce>0){this.modal.enableButton("Start")}else{this.modal.disableButton("Start")}if(this.counter.running>0){this.modal.enableButton("Pause")}else{this.modal.disableButton("Pause")}if(this.counter.running==0){this.modal.enableButton("Choose local file");this.modal.enableButton("Choose FTP file");this.modal.enableButton("Create new file")}else{this.modal.disableButton("Choose local file");this.modal.disableButton("Choose FTP file");this.modal.disableButton("Create new file")}if(this.current_user&&this.options.ftp_upload_dir&&this.options.ftp_upload_site){this.modal.showButton("Choose FTP file")}else{this.modal.hideButton("Choose FTP file")}if(this.counter.announce+this.counter.success+this.counter.error>0){$(this.el).find("#upload-table").show()}else{$(this.el).find("#upload-table").hide()}},_upload_percentage:function(h,i){return(this.upload_completed+(h*i))/this.upload_size},_template:function(i,h){return'<div id="'+i+'" class="upload-box"><table id="upload-table" class="table table-striped" style="display: none;"><thead><tr><th>Name</th><th>Size</th><th>Type</th><th>Genome</th><th>Settings</th><th>Status</th><th></th></tr></thead><tbody></tbody></table></div><h6 id="'+h+'" class="upload-info"></h6>'}})}); \ No newline at end of file +define(["galaxy.modal","utils/utils","mvc/upload/upload-button","mvc/upload/upload-model","mvc/upload/upload-row","mvc/upload/upload-ftp","mvc/ui/ui-popover","mvc/ui","utils/uploadbox"],function(a,f,e,c,b,g,d){return Backbone.View.extend({options:{nginx_upload_path:""},modal:null,ui_button:null,uploadbox:null,current_history:null,upload_size:0,list_extensions:[],list_genomes:[],auto:{id:"auto",text:"Auto-detect",description:"This system will try to detect the file type automatically. If your file is not detected properly as one of the known formats, it most likely means that it has some format problems (e.g., different number of columns on different rows). You can still coerce the system to set your data to the format you think it should be. You can also upload compressed files, which will automatically be decompressed."},collection:new c.Collection(),ftp:null,counter:{announce:0,success:0,error:0,running:0,reset:function(){this.announce=this.success=this.error=this.running=0}},initialize:function(i){var h=this;if(i){this.options=_.defaults(i,this.options)}if(!Galaxy.currHistoryPanel||!Galaxy.currHistoryPanel.model){window.setTimeout(function(){h.initialize()},500);return}this.ui_button=new e.Model({icon:"fa-upload",tooltip:"Download from URL or upload files from disk",label:"Load Data",onclick:function(j){if(j){h._eventShow(j)}},onunload:function(){if(h.counter.running>0){return"Several uploads are still processing."}}});$("#left .unified-panel-header-inner").append((new e.View(this.ui_button)).$el);var h=this;f.jsonFromUrl(galaxy_config.root+"api/datatypes?extension_only=False",function(j){for(key in j){h.list_extensions.push({id:j[key].extension,text:j[key].extension,description:j[key].description,description_url:j[key].description_url})}h.list_extensions.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0});if(!h.options.datatypes_disable_auto){h.list_extensions.unshift(h.auto)}});f.jsonFromUrl(galaxy_config.root+"api/genomes",function(j){for(key in j){h.list_genomes.push({id:j[key][1],text:j[key][0]})}h.list_genomes.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0})});this.collection.on("remove",function(j){h._eventRemove(j)});this.collection.on("change:genome",function(k){var j=k.get("genome");h.collection.each(function(l){if(l.get("status")=="init"&&l.get("genome")=="?"){l.set("genome",j)}})})},_eventShow:function(j){j.preventDefault();if(!this.modal){var h=this;this.modal=new a.GalaxyModal({title:"Download data directly from web or upload files from your disk",body:this._template("upload-box","upload-info"),buttons:{"Choose local file":function(){h.uploadbox.select()},"Choose FTP file":function(){h._eventFtp()},"Paste/Fetch data":function(){h._eventCreate()},Start:function(){h._eventStart()},Pause:function(){h._eventStop()},Reset:function(){h._eventReset()},Close:function(){h.modal.hide()},},height:"400",width:"900",closing_events:true});this.setElement("#upload-box");var h=this;this.uploadbox=this.$el.uploadbox({announce:function(k,l,m){h._eventAnnounce(k,l,m)},initialize:function(k,l,m){return h._eventInitialize(k,l,m)},progress:function(k,l,m){h._eventProgress(k,l,m)},success:function(k,l,m){h._eventSuccess(k,l,m)},error:function(k,l,m){h._eventError(k,l,m)},complete:function(){h._eventComplete()}});var i=this.modal.getButton("Choose FTP file");this.ftp=new d.View({title:"FTP files",container:i})}this.modal.show();this._updateUser();this._updateScreen()},_eventRemove:function(i){var h=i.get("status");if(h=="success"){this.counter.success--}else{if(h=="error"){this.counter.error--}else{this.counter.announce--}}this._updateScreen();this.uploadbox.remove(i.id)},_eventAnnounce:function(h,i,k){this.counter.announce++;this._updateScreen();var j=new b(this,{id:h,file_name:i.name,file_size:i.size,file_mode:i.mode,file_path:i.path});this.collection.add(j.model);$(this.el).find("tbody:first").append(j.$el);j.render()},_eventInitialize:function(m,j,s){var k=this.collection.get(m);k.set("status","running");var o=k.get("file_name");var n=k.get("file_path");var h=k.get("file_mode");var p=k.get("extension");var r=k.get("genome");var q=k.get("url_paste");var l=k.get("space_to_tabs");var i=k.get("to_posix_lines");if(!q&&!(j.size>0)){return null}this.uploadbox.configure({url:this.options.nginx_upload_path});if(h=="local"){this.uploadbox.configure({paramname:"files_0|file_data"})}else{this.uploadbox.configure({paramname:null})}tool_input={};if(h=="new"){tool_input["files_0|url_paste"]=q}if(h=="ftp"){tool_input["files_0|ftp_files"]=n}tool_input.dbkey=r;tool_input.file_type=p;tool_input["files_0|type"]="upload_dataset";tool_input.space_to_tabs=l;tool_input.to_posix_lines=i;data={};data.history_id=this.current_history;data.tool_id="upload1";data.inputs=JSON.stringify(tool_input);return data},_eventProgress:function(i,j,h){var k=this.collection.get(i);k.set("percentage",h);this.ui_button.set("percentage",this._upload_percentage(h,j.size))},_eventSuccess:function(i,j,l){var k=this.collection.get(i);k.set("percentage",100);k.set("status","success");var h=k.get("file_size");this.ui_button.set("percentage",this._upload_percentage(100,h));this.upload_completed+=h*100;this.counter.announce--;this.counter.success++;this._updateScreen();Galaxy.currHistoryPanel.refreshHdas()},_eventError:function(h,i,k){var j=this.collection.get(h);j.set("percentage",100);j.set("status","error");j.set("info",k);this.ui_button.set("percentage",this._upload_percentage(100,i.size));this.ui_button.set("status","danger");this.upload_completed+=i.size*100;this.counter.announce--;this.counter.error++;this._updateScreen()},_eventComplete:function(){this.collection.each(function(h){if(h.get("status")=="queued"){h.set("status","init")}});this.counter.running=0;this._updateScreen()},_eventFtp:function(){if(!this.ftp.visible){this.ftp.empty();this.ftp.append((new g(this)).$el);this.ftp.show()}else{this.ftp.hide()}},_eventCreate:function(){this.uploadbox.add([{name:"New File",size:0,mode:"new"}])},_eventStart:function(){if(this.counter.announce==0||this.counter.running>0){return}var h=this;this.upload_size=0;this.upload_completed=0;this.collection.each(function(i){if(i.get("status")=="init"){i.set("status","queued");h.upload_size+=i.get("file_size")}});this.ui_button.set("percentage",0);this.ui_button.set("status","success");this.counter.running=this.counter.announce;this._updateScreen();this.uploadbox.start()},_eventStop:function(){if(this.counter.running==0){return}this.ui_button.set("status","info");this.uploadbox.stop();$("#upload-info").html("Queue will pause after completing the current file...")},_eventReset:function(){if(this.counter.running==0){this.collection.reset();this.counter.reset();this._updateScreen();this.uploadbox.reset();this.ui_button.set("percentage",0)}},_updateUser:function(){this.current_user=Galaxy.currUser.get("id");this.current_history=null;if(this.current_user){this.current_history=Galaxy.currHistoryPanel.model.get("id")}},_updateScreen:function(){if(this.counter.announce==0){if(this.uploadbox.compatible()){message="You can Drag & Drop files into this box."}else{message="Unfortunately, your browser does not support multiple file uploads or drag&drop.<br>Some supported browsers are: Firefox 4+, Chrome 7+, IE 10+, Opera 12+ or Safari 6+."}}else{if(this.counter.running==0){message="You added "+this.counter.announce+" file(s) to the queue. Add more files or click 'Start' to proceed."}else{message="Please wait..."+this.counter.announce+" out of "+this.counter.running+" remaining."}}$("#upload-info").html(message);if(this.counter.running==0&&this.counter.announce+this.counter.success+this.counter.error>0){this.modal.enableButton("Reset")}else{this.modal.disableButton("Reset")}if(this.counter.running==0&&this.counter.announce>0){this.modal.enableButton("Start")}else{this.modal.disableButton("Start")}if(this.counter.running>0){this.modal.enableButton("Pause")}else{this.modal.disableButton("Pause")}if(this.counter.running==0){this.modal.enableButton("Choose local file");this.modal.enableButton("Choose FTP file");this.modal.enableButton("Paste/Fetch data")}else{this.modal.disableButton("Choose local file");this.modal.disableButton("Choose FTP file");this.modal.disableButton("Paste/Fetch data")}if(this.current_user&&this.options.ftp_upload_dir&&this.options.ftp_upload_site){this.modal.showButton("Choose FTP file")}else{this.modal.hideButton("Choose FTP file")}if(this.counter.announce+this.counter.success+this.counter.error>0){$(this.el).find("#upload-table").show()}else{$(this.el).find("#upload-table").hide()}},_upload_percentage:function(h,i){return(this.upload_completed+(h*i))/this.upload_size},_template:function(i,h){return'<div id="'+i+'" class="upload-box"><table id="upload-table" class="table table-striped" style="display: none;"><thead><tr><th>Name</th><th>Size</th><th>Type</th><th>Genome</th><th>Settings</th><th>Status</th><th></th></tr></thead><tbody></tbody></table></div><h6 id="'+h+'" class="upload-info"></h6>'}})}); \ No newline at end of file diff -r c755326c2018efa988daebe96c355a32b6482018 -r 26e37f6af3a8a090d9d6e923fa3e90acb0e64572 tools/data_source/upload.xml --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,7 @@ <?xml version="1.0"?><tool name="Upload File" id="upload1" version="1.1.4" workflow_compatible="false"> + <hidden>True</hidden><description> from your computer </description> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Upload: Load user details on show
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c755326c2018/ Changeset: c755326c2018 User: guerler Date: 2014-02-18 19:21:03 Summary: Upload: Load user details on show Affected #: 2 files diff -r 5ea343bbc97bc1ebb8fa1bcd3e0df2889ef232ca -r c755326c2018efa988daebe96c355a32b6482018 static/scripts/mvc/upload/upload-view.js --- a/static/scripts/mvc/upload/upload-view.js +++ b/static/scripts/mvc/upload/upload-view.js @@ -214,13 +214,16 @@ title : 'FTP files', container : button }); - - // setup info - this._updateScreen(); } // show modal this.modal.show(); + + // refresh + this._updateUser(); + + // setup info + this._updateScreen(); }, // @@ -526,9 +529,6 @@ // set screen _updateScreen: function () { - // refresh - this._updateUser(); - /* update on screen info */ diff -r 5ea343bbc97bc1ebb8fa1bcd3e0df2889ef232ca -r c755326c2018efa988daebe96c355a32b6482018 static/scripts/packed/mvc/upload/upload-view.js --- a/static/scripts/packed/mvc/upload/upload-view.js +++ b/static/scripts/packed/mvc/upload/upload-view.js @@ -1,1 +1,1 @@ -define(["galaxy.modal","utils/utils","mvc/upload/upload-button","mvc/upload/upload-model","mvc/upload/upload-row","mvc/upload/upload-ftp","mvc/ui/ui-popover","mvc/ui","utils/uploadbox"],function(a,f,e,c,b,g,d){return Backbone.View.extend({options:{nginx_upload_path:""},modal:null,ui_button:null,uploadbox:null,current_history:null,upload_size:0,list_extensions:[],list_genomes:[],auto:{id:"auto",text:"Auto-detect",description:"This system will try to detect the file type automatically. If your file is not detected properly as one of the known formats, it most likely means that it has some format problems (e.g., different number of columns on different rows). You can still coerce the system to set your data to the format you think it should be. You can also upload compressed files, which will automatically be decompressed."},collection:new c.Collection(),ftp:null,counter:{announce:0,success:0,error:0,running:0,reset:function(){this.announce=this.success=this.error=this.running=0}},initialize:function(i){var h=this;if(i){this.options=_.defaults(i,this.options)}if(!Galaxy.currHistoryPanel||!Galaxy.currHistoryPanel.model){window.setTimeout(function(){h.initialize()},500);return}this.ui_button=new e.Model({icon:"fa-upload",tooltip:"Download from URL or upload files from disk",label:"Load Data",onclick:function(j){if(j){h._eventShow(j)}},onunload:function(){if(h.counter.running>0){return"Several uploads are still processing."}}});$("#left .unified-panel-header-inner").append((new e.View(this.ui_button)).$el);var h=this;f.jsonFromUrl(galaxy_config.root+"api/datatypes?extension_only=False",function(j){for(key in j){h.list_extensions.push({id:j[key].extension,text:j[key].extension,description:j[key].description,description_url:j[key].description_url})}h.list_extensions.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0});if(!h.options.datatypes_disable_auto){h.list_extensions.unshift(h.auto)}});f.jsonFromUrl(galaxy_config.root+"api/genomes",function(j){for(key in j){h.list_genomes.push({id:j[key][1],text:j[key][0]})}h.list_genomes.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0})});this.collection.on("remove",function(j){h._eventRemove(j)});this.collection.on("change:genome",function(k){var j=k.get("genome");h.collection.each(function(l){if(l.get("status")=="init"&&l.get("genome")=="?"){l.set("genome",j)}})})},_eventShow:function(j){j.preventDefault();if(!this.modal){var h=this;this.modal=new a.GalaxyModal({title:"Download data directly from web or upload files from your disk",body:this._template("upload-box","upload-info"),buttons:{"Choose local file":function(){h.uploadbox.select()},"Choose FTP file":function(){h._eventFtp()},"Create new file":function(){h._eventCreate()},Start:function(){h._eventStart()},Pause:function(){h._eventStop()},Reset:function(){h._eventReset()},Close:function(){h.modal.hide()},},height:"400",width:"900",closing_events:true});this.setElement("#upload-box");var h=this;this.uploadbox=this.$el.uploadbox({announce:function(k,l,m){h._eventAnnounce(k,l,m)},initialize:function(k,l,m){return h._eventInitialize(k,l,m)},progress:function(k,l,m){h._eventProgress(k,l,m)},success:function(k,l,m){h._eventSuccess(k,l,m)},error:function(k,l,m){h._eventError(k,l,m)},complete:function(){h._eventComplete()}});var i=this.modal.getButton("Choose FTP file");this.ftp=new d.View({title:"FTP files",container:i});this._updateScreen()}this.modal.show()},_eventRemove:function(i){var h=i.get("status");if(h=="success"){this.counter.success--}else{if(h=="error"){this.counter.error--}else{this.counter.announce--}}this._updateScreen();this.uploadbox.remove(i.id)},_eventAnnounce:function(h,i,k){this.counter.announce++;this._updateScreen();var j=new b(this,{id:h,file_name:i.name,file_size:i.size,file_mode:i.mode,file_path:i.path});this.collection.add(j.model);$(this.el).find("tbody:first").append(j.$el);j.render()},_eventInitialize:function(m,j,s){var k=this.collection.get(m);k.set("status","running");var o=k.get("file_name");var n=k.get("file_path");var h=k.get("file_mode");var p=k.get("extension");var r=k.get("genome");var q=k.get("url_paste");var l=k.get("space_to_tabs");var i=k.get("to_posix_lines");if(!q&&!(j.size>0)){return null}this.uploadbox.configure({url:this.options.nginx_upload_path});if(h=="local"){this.uploadbox.configure({paramname:"files_0|file_data"})}else{this.uploadbox.configure({paramname:null})}tool_input={};if(h=="new"){tool_input["files_0|url_paste"]=q}if(h=="ftp"){tool_input["files_0|ftp_files"]=n}tool_input.dbkey=r;tool_input.file_type=p;tool_input["files_0|type"]="upload_dataset";tool_input.space_to_tabs=l;tool_input.to_posix_lines=i;data={};data.history_id=this.current_history;data.tool_id="upload1";data.inputs=JSON.stringify(tool_input);return data},_eventProgress:function(i,j,h){var k=this.collection.get(i);k.set("percentage",h);this.ui_button.set("percentage",this._upload_percentage(h,j.size))},_eventSuccess:function(i,j,l){var k=this.collection.get(i);k.set("percentage",100);k.set("status","success");var h=k.get("file_size");this.ui_button.set("percentage",this._upload_percentage(100,h));this.upload_completed+=h*100;this.counter.announce--;this.counter.success++;this._updateScreen();Galaxy.currHistoryPanel.refreshHdas()},_eventError:function(h,i,k){var j=this.collection.get(h);j.set("percentage",100);j.set("status","error");j.set("info",k);this.ui_button.set("percentage",this._upload_percentage(100,i.size));this.ui_button.set("status","danger");this.upload_completed+=i.size*100;this.counter.announce--;this.counter.error++;this._updateScreen()},_eventComplete:function(){this.collection.each(function(h){if(h.get("status")=="queued"){h.set("status","init")}});this.counter.running=0;this._updateScreen()},_eventFtp:function(){if(!this.ftp.visible){this.ftp.empty();this.ftp.append((new g(this)).$el);this.ftp.show()}else{this.ftp.hide()}},_eventCreate:function(){this.uploadbox.add([{name:"New File",size:0,mode:"new"}])},_eventStart:function(){if(this.counter.announce==0||this.counter.running>0){return}var h=this;this.upload_size=0;this.upload_completed=0;this.collection.each(function(i){if(i.get("status")=="init"){i.set("status","queued");h.upload_size+=i.get("file_size")}});this.ui_button.set("percentage",0);this.ui_button.set("status","success");this.counter.running=this.counter.announce;this._updateScreen();this.uploadbox.start()},_eventStop:function(){if(this.counter.running==0){return}this.ui_button.set("status","info");this.uploadbox.stop();$("#upload-info").html("Queue will pause after completing the current file...")},_eventReset:function(){if(this.counter.running==0){this.collection.reset();this.counter.reset();this._updateScreen();this.uploadbox.reset();this.ui_button.set("percentage",0)}},_updateUser:function(){this.current_user=Galaxy.currUser.get("id");this.current_history=null;if(this.current_user){this.current_history=Galaxy.currHistoryPanel.model.get("id")}},_updateScreen:function(){this._updateUser();if(this.counter.announce==0){if(this.uploadbox.compatible()){message="You can Drag & Drop files into this box."}else{message="Unfortunately, your browser does not support multiple file uploads or drag&drop.<br>Some supported browsers are: Firefox 4+, Chrome 7+, IE 10+, Opera 12+ or Safari 6+."}}else{if(this.counter.running==0){message="You added "+this.counter.announce+" file(s) to the queue. Add more files or click 'Start' to proceed."}else{message="Please wait..."+this.counter.announce+" out of "+this.counter.running+" remaining."}}$("#upload-info").html(message);if(this.counter.running==0&&this.counter.announce+this.counter.success+this.counter.error>0){this.modal.enableButton("Reset")}else{this.modal.disableButton("Reset")}if(this.counter.running==0&&this.counter.announce>0){this.modal.enableButton("Start")}else{this.modal.disableButton("Start")}if(this.counter.running>0){this.modal.enableButton("Pause")}else{this.modal.disableButton("Pause")}if(this.counter.running==0){this.modal.enableButton("Choose local file");this.modal.enableButton("Choose FTP file");this.modal.enableButton("Create new file")}else{this.modal.disableButton("Choose local file");this.modal.disableButton("Choose FTP file");this.modal.disableButton("Create new file")}if(this.current_user&&this.options.ftp_upload_dir&&this.options.ftp_upload_site){this.modal.showButton("Choose FTP file")}else{this.modal.hideButton("Choose FTP file")}if(this.counter.announce+this.counter.success+this.counter.error>0){$(this.el).find("#upload-table").show()}else{$(this.el).find("#upload-table").hide()}},_upload_percentage:function(h,i){return(this.upload_completed+(h*i))/this.upload_size},_template:function(i,h){return'<div id="'+i+'" class="upload-box"><table id="upload-table" class="table table-striped" style="display: none;"><thead><tr><th>Name</th><th>Size</th><th>Type</th><th>Genome</th><th>Settings</th><th>Status</th><th></th></tr></thead><tbody></tbody></table></div><h6 id="'+h+'" class="upload-info"></h6>'}})}); \ No newline at end of file +define(["galaxy.modal","utils/utils","mvc/upload/upload-button","mvc/upload/upload-model","mvc/upload/upload-row","mvc/upload/upload-ftp","mvc/ui/ui-popover","mvc/ui","utils/uploadbox"],function(a,f,e,c,b,g,d){return Backbone.View.extend({options:{nginx_upload_path:""},modal:null,ui_button:null,uploadbox:null,current_history:null,upload_size:0,list_extensions:[],list_genomes:[],auto:{id:"auto",text:"Auto-detect",description:"This system will try to detect the file type automatically. If your file is not detected properly as one of the known formats, it most likely means that it has some format problems (e.g., different number of columns on different rows). You can still coerce the system to set your data to the format you think it should be. You can also upload compressed files, which will automatically be decompressed."},collection:new c.Collection(),ftp:null,counter:{announce:0,success:0,error:0,running:0,reset:function(){this.announce=this.success=this.error=this.running=0}},initialize:function(i){var h=this;if(i){this.options=_.defaults(i,this.options)}if(!Galaxy.currHistoryPanel||!Galaxy.currHistoryPanel.model){window.setTimeout(function(){h.initialize()},500);return}this.ui_button=new e.Model({icon:"fa-upload",tooltip:"Download from URL or upload files from disk",label:"Load Data",onclick:function(j){if(j){h._eventShow(j)}},onunload:function(){if(h.counter.running>0){return"Several uploads are still processing."}}});$("#left .unified-panel-header-inner").append((new e.View(this.ui_button)).$el);var h=this;f.jsonFromUrl(galaxy_config.root+"api/datatypes?extension_only=False",function(j){for(key in j){h.list_extensions.push({id:j[key].extension,text:j[key].extension,description:j[key].description,description_url:j[key].description_url})}h.list_extensions.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0});if(!h.options.datatypes_disable_auto){h.list_extensions.unshift(h.auto)}});f.jsonFromUrl(galaxy_config.root+"api/genomes",function(j){for(key in j){h.list_genomes.push({id:j[key][1],text:j[key][0]})}h.list_genomes.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0})});this.collection.on("remove",function(j){h._eventRemove(j)});this.collection.on("change:genome",function(k){var j=k.get("genome");h.collection.each(function(l){if(l.get("status")=="init"&&l.get("genome")=="?"){l.set("genome",j)}})})},_eventShow:function(j){j.preventDefault();if(!this.modal){var h=this;this.modal=new a.GalaxyModal({title:"Download data directly from web or upload files from your disk",body:this._template("upload-box","upload-info"),buttons:{"Choose local file":function(){h.uploadbox.select()},"Choose FTP file":function(){h._eventFtp()},"Create new file":function(){h._eventCreate()},Start:function(){h._eventStart()},Pause:function(){h._eventStop()},Reset:function(){h._eventReset()},Close:function(){h.modal.hide()},},height:"400",width:"900",closing_events:true});this.setElement("#upload-box");var h=this;this.uploadbox=this.$el.uploadbox({announce:function(k,l,m){h._eventAnnounce(k,l,m)},initialize:function(k,l,m){return h._eventInitialize(k,l,m)},progress:function(k,l,m){h._eventProgress(k,l,m)},success:function(k,l,m){h._eventSuccess(k,l,m)},error:function(k,l,m){h._eventError(k,l,m)},complete:function(){h._eventComplete()}});var i=this.modal.getButton("Choose FTP file");this.ftp=new d.View({title:"FTP files",container:i})}this.modal.show();this._updateUser();this._updateScreen()},_eventRemove:function(i){var h=i.get("status");if(h=="success"){this.counter.success--}else{if(h=="error"){this.counter.error--}else{this.counter.announce--}}this._updateScreen();this.uploadbox.remove(i.id)},_eventAnnounce:function(h,i,k){this.counter.announce++;this._updateScreen();var j=new b(this,{id:h,file_name:i.name,file_size:i.size,file_mode:i.mode,file_path:i.path});this.collection.add(j.model);$(this.el).find("tbody:first").append(j.$el);j.render()},_eventInitialize:function(m,j,s){var k=this.collection.get(m);k.set("status","running");var o=k.get("file_name");var n=k.get("file_path");var h=k.get("file_mode");var p=k.get("extension");var r=k.get("genome");var q=k.get("url_paste");var l=k.get("space_to_tabs");var i=k.get("to_posix_lines");if(!q&&!(j.size>0)){return null}this.uploadbox.configure({url:this.options.nginx_upload_path});if(h=="local"){this.uploadbox.configure({paramname:"files_0|file_data"})}else{this.uploadbox.configure({paramname:null})}tool_input={};if(h=="new"){tool_input["files_0|url_paste"]=q}if(h=="ftp"){tool_input["files_0|ftp_files"]=n}tool_input.dbkey=r;tool_input.file_type=p;tool_input["files_0|type"]="upload_dataset";tool_input.space_to_tabs=l;tool_input.to_posix_lines=i;data={};data.history_id=this.current_history;data.tool_id="upload1";data.inputs=JSON.stringify(tool_input);return data},_eventProgress:function(i,j,h){var k=this.collection.get(i);k.set("percentage",h);this.ui_button.set("percentage",this._upload_percentage(h,j.size))},_eventSuccess:function(i,j,l){var k=this.collection.get(i);k.set("percentage",100);k.set("status","success");var h=k.get("file_size");this.ui_button.set("percentage",this._upload_percentage(100,h));this.upload_completed+=h*100;this.counter.announce--;this.counter.success++;this._updateScreen();Galaxy.currHistoryPanel.refreshHdas()},_eventError:function(h,i,k){var j=this.collection.get(h);j.set("percentage",100);j.set("status","error");j.set("info",k);this.ui_button.set("percentage",this._upload_percentage(100,i.size));this.ui_button.set("status","danger");this.upload_completed+=i.size*100;this.counter.announce--;this.counter.error++;this._updateScreen()},_eventComplete:function(){this.collection.each(function(h){if(h.get("status")=="queued"){h.set("status","init")}});this.counter.running=0;this._updateScreen()},_eventFtp:function(){if(!this.ftp.visible){this.ftp.empty();this.ftp.append((new g(this)).$el);this.ftp.show()}else{this.ftp.hide()}},_eventCreate:function(){this.uploadbox.add([{name:"New File",size:0,mode:"new"}])},_eventStart:function(){if(this.counter.announce==0||this.counter.running>0){return}var h=this;this.upload_size=0;this.upload_completed=0;this.collection.each(function(i){if(i.get("status")=="init"){i.set("status","queued");h.upload_size+=i.get("file_size")}});this.ui_button.set("percentage",0);this.ui_button.set("status","success");this.counter.running=this.counter.announce;this._updateScreen();this.uploadbox.start()},_eventStop:function(){if(this.counter.running==0){return}this.ui_button.set("status","info");this.uploadbox.stop();$("#upload-info").html("Queue will pause after completing the current file...")},_eventReset:function(){if(this.counter.running==0){this.collection.reset();this.counter.reset();this._updateScreen();this.uploadbox.reset();this.ui_button.set("percentage",0)}},_updateUser:function(){this.current_user=Galaxy.currUser.get("id");this.current_history=null;if(this.current_user){this.current_history=Galaxy.currHistoryPanel.model.get("id")}},_updateScreen:function(){if(this.counter.announce==0){if(this.uploadbox.compatible()){message="You can Drag & Drop files into this box."}else{message="Unfortunately, your browser does not support multiple file uploads or drag&drop.<br>Some supported browsers are: Firefox 4+, Chrome 7+, IE 10+, Opera 12+ or Safari 6+."}}else{if(this.counter.running==0){message="You added "+this.counter.announce+" file(s) to the queue. Add more files or click 'Start' to proceed."}else{message="Please wait..."+this.counter.announce+" out of "+this.counter.running+" remaining."}}$("#upload-info").html(message);if(this.counter.running==0&&this.counter.announce+this.counter.success+this.counter.error>0){this.modal.enableButton("Reset")}else{this.modal.disableButton("Reset")}if(this.counter.running==0&&this.counter.announce>0){this.modal.enableButton("Start")}else{this.modal.disableButton("Start")}if(this.counter.running>0){this.modal.enableButton("Pause")}else{this.modal.disableButton("Pause")}if(this.counter.running==0){this.modal.enableButton("Choose local file");this.modal.enableButton("Choose FTP file");this.modal.enableButton("Create new file")}else{this.modal.disableButton("Choose local file");this.modal.disableButton("Choose FTP file");this.modal.disableButton("Create new file")}if(this.current_user&&this.options.ftp_upload_dir&&this.options.ftp_upload_site){this.modal.showButton("Choose FTP file")}else{this.modal.hideButton("Choose FTP file")}if(this.counter.announce+this.counter.success+this.counter.error>0){$(this.el).find("#upload-table").show()}else{$(this.el).find("#upload-table").hide()}},_upload_percentage:function(h,i){return(this.upload_completed+(h*i))/this.upload_size},_template:function(i,h){return'<div id="'+i+'" class="upload-box"><table id="upload-table" class="table table-striped" style="display: none;"><thead><tr><th>Name</th><th>Size</th><th>Type</th><th>Genome</th><th>Settings</th><th>Status</th><th></th></tr></thead><tbody></tbody></table></div><h6 id="'+h+'" class="upload-info"></h6>'}})}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Upload: Make upload available to anonymous users
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5ea343bbc97b/ Changeset: 5ea343bbc97b User: guerler Date: 2014-02-18 19:13:04 Summary: Upload: Make upload available to anonymous users Affected #: 5 files diff -r 05c93f3b424c372c14468f9aa25a79db88bfca24 -r 5ea343bbc97bc1ebb8fa1bcd3e0df2889ef232ca lib/galaxy/visualization/genomes.py --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -218,9 +218,10 @@ # Add user's custom keys to dbkeys. user_keys_dict = {} user = trans.get_user() - if 'dbkeys' in user.preferences: - user_keys_dict = from_json_string( user.preferences[ 'dbkeys' ] ) - dbkeys.extend( [ (attributes[ 'name' ], key ) for key, attributes in user_keys_dict.items() ] ) + if user: + if 'dbkeys' in user.preferences: + user_keys_dict = from_json_string( user.preferences[ 'dbkeys' ] ) + dbkeys.extend( [ (attributes[ 'name' ], key ) for key, attributes in user_keys_dict.items() ] ) # Add app keys to dbkeys. diff -r 05c93f3b424c372c14468f9aa25a79db88bfca24 -r 5ea343bbc97bc1ebb8fa1bcd3e0df2889ef232ca lib/galaxy/webapps/galaxy/api/genomes.py --- a/lib/galaxy/webapps/galaxy/api/genomes.py +++ b/lib/galaxy/webapps/galaxy/api/genomes.py @@ -14,7 +14,7 @@ RESTful controller for interactions with genome data. """ - @web.expose_api + @web.expose_api_anonymous def index( self, trans, **kwd ): """ GET /api/genomes: returns a list of installed genomes diff -r 05c93f3b424c372c14468f9aa25a79db88bfca24 -r 5ea343bbc97bc1ebb8fa1bcd3e0df2889ef232ca lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -64,7 +64,7 @@ trans.response.status = 500 return { 'error': str( exc ) } - @web.expose_api + @web.expose_api_anonymous def create( self, trans, payload, **kwd ): """ POST /api/tools diff -r 05c93f3b424c372c14468f9aa25a79db88bfca24 -r 5ea343bbc97bc1ebb8fa1bcd3e0df2889ef232ca static/scripts/mvc/upload/upload-view.js --- a/static/scripts/mvc/upload/upload-view.js +++ b/static/scripts/mvc/upload/upload-view.js @@ -90,11 +90,6 @@ return; } - // check if logged in - if (!Galaxy.currUser.get('id')) { - return; - } - // create model this.ui_button = new UploadButton.Model({ icon : 'fa-upload', @@ -213,19 +208,15 @@ complete : function() { self._eventComplete() } }); + // add ftp file viewer + var button = this.modal.getButton('Choose FTP file'); + this.ftp = new Popover.View({ + title : 'FTP files', + container : button + }); + // setup info this._updateScreen(); - - // add ftp file viewer - if (this.options.ftp_upload_dir && this.options.ftp_upload_site) { - var button = this.modal.getButton('Choose FTP file'); - this.ftp = new Popover.View({ - title : 'FTP files', - container : button - }); - } else { - this.modal.hideButton('Choose FTP file'); - } } // show modal @@ -429,7 +420,7 @@ // // events triggered by this view // - + _eventFtp: function() { // check if popover is visible if (!this.ftp.visible) { @@ -476,9 +467,6 @@ this.ui_button.set('percentage', 0); this.ui_button.set('status', 'success'); - // backup current history - this.current_history = Galaxy.currHistoryPanel.model.get('id'); - // update running this.counter.running = this.counter.announce; this._updateScreen(); @@ -526,8 +514,21 @@ } }, + // update uset + _updateUser: function() { + // backup current history + this.current_user = Galaxy.currUser.get('id'); + this.current_history = null; + if (this.current_user) { + this.current_history = Galaxy.currHistoryPanel.model.get('id'); + } + }, + // set screen _updateScreen: function () { + // refresh + this._updateUser(); + /* update on screen info */ @@ -583,6 +584,13 @@ this.modal.disableButton('Create new file'); } + // ftp button + if (this.current_user && this.options.ftp_upload_dir && this.options.ftp_upload_site) { + this.modal.showButton('Choose FTP file'); + } else { + this.modal.hideButton('Choose FTP file'); + } + // table visibility if (this.counter.announce + this.counter.success + this.counter.error > 0) $(this.el).find('#upload-table').show(); diff -r 05c93f3b424c372c14468f9aa25a79db88bfca24 -r 5ea343bbc97bc1ebb8fa1bcd3e0df2889ef232ca static/scripts/packed/mvc/upload/upload-view.js --- a/static/scripts/packed/mvc/upload/upload-view.js +++ b/static/scripts/packed/mvc/upload/upload-view.js @@ -1,1 +1,1 @@ -define(["galaxy.modal","utils/utils","mvc/upload/upload-button","mvc/upload/upload-model","mvc/upload/upload-row","mvc/upload/upload-ftp","mvc/ui/ui-popover","mvc/ui","utils/uploadbox"],function(a,f,e,c,b,g,d){return Backbone.View.extend({options:{nginx_upload_path:""},modal:null,ui_button:null,uploadbox:null,current_history:null,upload_size:0,list_extensions:[],list_genomes:[],auto:{id:"auto",text:"Auto-detect",description:"This system will try to detect the file type automatically. If your file is not detected properly as one of the known formats, it most likely means that it has some format problems (e.g., different number of columns on different rows). You can still coerce the system to set your data to the format you think it should be. You can also upload compressed files, which will automatically be decompressed."},collection:new c.Collection(),ftp:null,counter:{announce:0,success:0,error:0,running:0,reset:function(){this.announce=this.success=this.error=this.running=0}},initialize:function(i){var h=this;if(i){this.options=_.defaults(i,this.options)}if(!Galaxy.currHistoryPanel||!Galaxy.currHistoryPanel.model){window.setTimeout(function(){h.initialize()},500);return}if(!Galaxy.currUser.get("id")){return}this.ui_button=new e.Model({icon:"fa-upload",tooltip:"Download from URL or upload files from disk",label:"Load Data",onclick:function(j){if(j){h._eventShow(j)}},onunload:function(){if(h.counter.running>0){return"Several uploads are still processing."}}});$("#left .unified-panel-header-inner").append((new e.View(this.ui_button)).$el);var h=this;f.jsonFromUrl(galaxy_config.root+"api/datatypes?extension_only=False",function(j){for(key in j){h.list_extensions.push({id:j[key].extension,text:j[key].extension,description:j[key].description,description_url:j[key].description_url})}h.list_extensions.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0});if(!h.options.datatypes_disable_auto){h.list_extensions.unshift(h.auto)}});f.jsonFromUrl(galaxy_config.root+"api/genomes",function(j){for(key in j){h.list_genomes.push({id:j[key][1],text:j[key][0]})}h.list_genomes.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0})});this.collection.on("remove",function(j){h._eventRemove(j)});this.collection.on("change:genome",function(k){var j=k.get("genome");h.collection.each(function(l){if(l.get("status")=="init"&&l.get("genome")=="?"){l.set("genome",j)}})})},_eventShow:function(j){j.preventDefault();if(!this.modal){var h=this;this.modal=new a.GalaxyModal({title:"Download data directly from web or upload files from your disk",body:this._template("upload-box","upload-info"),buttons:{"Choose local file":function(){h.uploadbox.select()},"Choose FTP file":function(){h._eventFtp()},"Create new file":function(){h._eventCreate()},Start:function(){h._eventStart()},Pause:function(){h._eventStop()},Reset:function(){h._eventReset()},Close:function(){h.modal.hide()},},height:"400",width:"900",closing_events:true});this.setElement("#upload-box");var h=this;this.uploadbox=this.$el.uploadbox({announce:function(k,l,m){h._eventAnnounce(k,l,m)},initialize:function(k,l,m){return h._eventInitialize(k,l,m)},progress:function(k,l,m){h._eventProgress(k,l,m)},success:function(k,l,m){h._eventSuccess(k,l,m)},error:function(k,l,m){h._eventError(k,l,m)},complete:function(){h._eventComplete()}});this._updateScreen();if(this.options.ftp_upload_dir&&this.options.ftp_upload_site){var i=this.modal.getButton("Choose FTP file");this.ftp=new d.View({title:"FTP files",container:i})}else{this.modal.hideButton("Choose FTP file")}}this.modal.show()},_eventRemove:function(i){var h=i.get("status");if(h=="success"){this.counter.success--}else{if(h=="error"){this.counter.error--}else{this.counter.announce--}}this._updateScreen();this.uploadbox.remove(i.id)},_eventAnnounce:function(h,i,k){this.counter.announce++;this._updateScreen();var j=new b(this,{id:h,file_name:i.name,file_size:i.size,file_mode:i.mode,file_path:i.path});this.collection.add(j.model);$(this.el).find("tbody:first").append(j.$el);j.render()},_eventInitialize:function(m,j,s){var k=this.collection.get(m);k.set("status","running");var o=k.get("file_name");var n=k.get("file_path");var h=k.get("file_mode");var p=k.get("extension");var r=k.get("genome");var q=k.get("url_paste");var l=k.get("space_to_tabs");var i=k.get("to_posix_lines");if(!q&&!(j.size>0)){return null}this.uploadbox.configure({url:this.options.nginx_upload_path});if(h=="local"){this.uploadbox.configure({paramname:"files_0|file_data"})}else{this.uploadbox.configure({paramname:null})}tool_input={};if(h=="new"){tool_input["files_0|url_paste"]=q}if(h=="ftp"){tool_input["files_0|ftp_files"]=n}tool_input.dbkey=r;tool_input.file_type=p;tool_input["files_0|type"]="upload_dataset";tool_input.space_to_tabs=l;tool_input.to_posix_lines=i;data={};data.history_id=this.current_history;data.tool_id="upload1";data.inputs=JSON.stringify(tool_input);return data},_eventProgress:function(i,j,h){var k=this.collection.get(i);k.set("percentage",h);this.ui_button.set("percentage",this._upload_percentage(h,j.size))},_eventSuccess:function(i,j,l){var k=this.collection.get(i);k.set("percentage",100);k.set("status","success");var h=k.get("file_size");this.ui_button.set("percentage",this._upload_percentage(100,h));this.upload_completed+=h*100;this.counter.announce--;this.counter.success++;this._updateScreen();Galaxy.currHistoryPanel.refreshHdas()},_eventError:function(h,i,k){var j=this.collection.get(h);j.set("percentage",100);j.set("status","error");j.set("info",k);this.ui_button.set("percentage",this._upload_percentage(100,i.size));this.ui_button.set("status","danger");this.upload_completed+=i.size*100;this.counter.announce--;this.counter.error++;this._updateScreen()},_eventComplete:function(){this.collection.each(function(h){if(h.get("status")=="queued"){h.set("status","init")}});this.counter.running=0;this._updateScreen()},_eventFtp:function(){if(!this.ftp.visible){this.ftp.empty();this.ftp.append((new g(this)).$el);this.ftp.show()}else{this.ftp.hide()}},_eventCreate:function(){this.uploadbox.add([{name:"New File",size:0,mode:"new"}])},_eventStart:function(){if(this.counter.announce==0||this.counter.running>0){return}var h=this;this.upload_size=0;this.upload_completed=0;this.collection.each(function(i){if(i.get("status")=="init"){i.set("status","queued");h.upload_size+=i.get("file_size")}});this.ui_button.set("percentage",0);this.ui_button.set("status","success");this.current_history=Galaxy.currHistoryPanel.model.get("id");this.counter.running=this.counter.announce;this._updateScreen();this.uploadbox.start()},_eventStop:function(){if(this.counter.running==0){return}this.ui_button.set("status","info");this.uploadbox.stop();$("#upload-info").html("Queue will pause after completing the current file...")},_eventReset:function(){if(this.counter.running==0){this.collection.reset();this.counter.reset();this._updateScreen();this.uploadbox.reset();this.ui_button.set("percentage",0)}},_updateScreen:function(){if(this.counter.announce==0){if(this.uploadbox.compatible()){message="You can Drag & Drop files into this box."}else{message="Unfortunately, your browser does not support multiple file uploads or drag&drop.<br>Some supported browsers are: Firefox 4+, Chrome 7+, IE 10+, Opera 12+ or Safari 6+."}}else{if(this.counter.running==0){message="You added "+this.counter.announce+" file(s) to the queue. Add more files or click 'Start' to proceed."}else{message="Please wait..."+this.counter.announce+" out of "+this.counter.running+" remaining."}}$("#upload-info").html(message);if(this.counter.running==0&&this.counter.announce+this.counter.success+this.counter.error>0){this.modal.enableButton("Reset")}else{this.modal.disableButton("Reset")}if(this.counter.running==0&&this.counter.announce>0){this.modal.enableButton("Start")}else{this.modal.disableButton("Start")}if(this.counter.running>0){this.modal.enableButton("Pause")}else{this.modal.disableButton("Pause")}if(this.counter.running==0){this.modal.enableButton("Choose local file");this.modal.enableButton("Choose FTP file");this.modal.enableButton("Create new file")}else{this.modal.disableButton("Choose local file");this.modal.disableButton("Choose FTP file");this.modal.disableButton("Create new file")}if(this.counter.announce+this.counter.success+this.counter.error>0){$(this.el).find("#upload-table").show()}else{$(this.el).find("#upload-table").hide()}},_upload_percentage:function(h,i){return(this.upload_completed+(h*i))/this.upload_size},_template:function(i,h){return'<div id="'+i+'" class="upload-box"><table id="upload-table" class="table table-striped" style="display: none;"><thead><tr><th>Name</th><th>Size</th><th>Type</th><th>Genome</th><th>Settings</th><th>Status</th><th></th></tr></thead><tbody></tbody></table></div><h6 id="'+h+'" class="upload-info"></h6>'}})}); \ No newline at end of file +define(["galaxy.modal","utils/utils","mvc/upload/upload-button","mvc/upload/upload-model","mvc/upload/upload-row","mvc/upload/upload-ftp","mvc/ui/ui-popover","mvc/ui","utils/uploadbox"],function(a,f,e,c,b,g,d){return Backbone.View.extend({options:{nginx_upload_path:""},modal:null,ui_button:null,uploadbox:null,current_history:null,upload_size:0,list_extensions:[],list_genomes:[],auto:{id:"auto",text:"Auto-detect",description:"This system will try to detect the file type automatically. If your file is not detected properly as one of the known formats, it most likely means that it has some format problems (e.g., different number of columns on different rows). You can still coerce the system to set your data to the format you think it should be. You can also upload compressed files, which will automatically be decompressed."},collection:new c.Collection(),ftp:null,counter:{announce:0,success:0,error:0,running:0,reset:function(){this.announce=this.success=this.error=this.running=0}},initialize:function(i){var h=this;if(i){this.options=_.defaults(i,this.options)}if(!Galaxy.currHistoryPanel||!Galaxy.currHistoryPanel.model){window.setTimeout(function(){h.initialize()},500);return}this.ui_button=new e.Model({icon:"fa-upload",tooltip:"Download from URL or upload files from disk",label:"Load Data",onclick:function(j){if(j){h._eventShow(j)}},onunload:function(){if(h.counter.running>0){return"Several uploads are still processing."}}});$("#left .unified-panel-header-inner").append((new e.View(this.ui_button)).$el);var h=this;f.jsonFromUrl(galaxy_config.root+"api/datatypes?extension_only=False",function(j){for(key in j){h.list_extensions.push({id:j[key].extension,text:j[key].extension,description:j[key].description,description_url:j[key].description_url})}h.list_extensions.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0});if(!h.options.datatypes_disable_auto){h.list_extensions.unshift(h.auto)}});f.jsonFromUrl(galaxy_config.root+"api/genomes",function(j){for(key in j){h.list_genomes.push({id:j[key][1],text:j[key][0]})}h.list_genomes.sort(function(l,k){return l.id>k.id?1:l.id<k.id?-1:0})});this.collection.on("remove",function(j){h._eventRemove(j)});this.collection.on("change:genome",function(k){var j=k.get("genome");h.collection.each(function(l){if(l.get("status")=="init"&&l.get("genome")=="?"){l.set("genome",j)}})})},_eventShow:function(j){j.preventDefault();if(!this.modal){var h=this;this.modal=new a.GalaxyModal({title:"Download data directly from web or upload files from your disk",body:this._template("upload-box","upload-info"),buttons:{"Choose local file":function(){h.uploadbox.select()},"Choose FTP file":function(){h._eventFtp()},"Create new file":function(){h._eventCreate()},Start:function(){h._eventStart()},Pause:function(){h._eventStop()},Reset:function(){h._eventReset()},Close:function(){h.modal.hide()},},height:"400",width:"900",closing_events:true});this.setElement("#upload-box");var h=this;this.uploadbox=this.$el.uploadbox({announce:function(k,l,m){h._eventAnnounce(k,l,m)},initialize:function(k,l,m){return h._eventInitialize(k,l,m)},progress:function(k,l,m){h._eventProgress(k,l,m)},success:function(k,l,m){h._eventSuccess(k,l,m)},error:function(k,l,m){h._eventError(k,l,m)},complete:function(){h._eventComplete()}});var i=this.modal.getButton("Choose FTP file");this.ftp=new d.View({title:"FTP files",container:i});this._updateScreen()}this.modal.show()},_eventRemove:function(i){var h=i.get("status");if(h=="success"){this.counter.success--}else{if(h=="error"){this.counter.error--}else{this.counter.announce--}}this._updateScreen();this.uploadbox.remove(i.id)},_eventAnnounce:function(h,i,k){this.counter.announce++;this._updateScreen();var j=new b(this,{id:h,file_name:i.name,file_size:i.size,file_mode:i.mode,file_path:i.path});this.collection.add(j.model);$(this.el).find("tbody:first").append(j.$el);j.render()},_eventInitialize:function(m,j,s){var k=this.collection.get(m);k.set("status","running");var o=k.get("file_name");var n=k.get("file_path");var h=k.get("file_mode");var p=k.get("extension");var r=k.get("genome");var q=k.get("url_paste");var l=k.get("space_to_tabs");var i=k.get("to_posix_lines");if(!q&&!(j.size>0)){return null}this.uploadbox.configure({url:this.options.nginx_upload_path});if(h=="local"){this.uploadbox.configure({paramname:"files_0|file_data"})}else{this.uploadbox.configure({paramname:null})}tool_input={};if(h=="new"){tool_input["files_0|url_paste"]=q}if(h=="ftp"){tool_input["files_0|ftp_files"]=n}tool_input.dbkey=r;tool_input.file_type=p;tool_input["files_0|type"]="upload_dataset";tool_input.space_to_tabs=l;tool_input.to_posix_lines=i;data={};data.history_id=this.current_history;data.tool_id="upload1";data.inputs=JSON.stringify(tool_input);return data},_eventProgress:function(i,j,h){var k=this.collection.get(i);k.set("percentage",h);this.ui_button.set("percentage",this._upload_percentage(h,j.size))},_eventSuccess:function(i,j,l){var k=this.collection.get(i);k.set("percentage",100);k.set("status","success");var h=k.get("file_size");this.ui_button.set("percentage",this._upload_percentage(100,h));this.upload_completed+=h*100;this.counter.announce--;this.counter.success++;this._updateScreen();Galaxy.currHistoryPanel.refreshHdas()},_eventError:function(h,i,k){var j=this.collection.get(h);j.set("percentage",100);j.set("status","error");j.set("info",k);this.ui_button.set("percentage",this._upload_percentage(100,i.size));this.ui_button.set("status","danger");this.upload_completed+=i.size*100;this.counter.announce--;this.counter.error++;this._updateScreen()},_eventComplete:function(){this.collection.each(function(h){if(h.get("status")=="queued"){h.set("status","init")}});this.counter.running=0;this._updateScreen()},_eventFtp:function(){if(!this.ftp.visible){this.ftp.empty();this.ftp.append((new g(this)).$el);this.ftp.show()}else{this.ftp.hide()}},_eventCreate:function(){this.uploadbox.add([{name:"New File",size:0,mode:"new"}])},_eventStart:function(){if(this.counter.announce==0||this.counter.running>0){return}var h=this;this.upload_size=0;this.upload_completed=0;this.collection.each(function(i){if(i.get("status")=="init"){i.set("status","queued");h.upload_size+=i.get("file_size")}});this.ui_button.set("percentage",0);this.ui_button.set("status","success");this.counter.running=this.counter.announce;this._updateScreen();this.uploadbox.start()},_eventStop:function(){if(this.counter.running==0){return}this.ui_button.set("status","info");this.uploadbox.stop();$("#upload-info").html("Queue will pause after completing the current file...")},_eventReset:function(){if(this.counter.running==0){this.collection.reset();this.counter.reset();this._updateScreen();this.uploadbox.reset();this.ui_button.set("percentage",0)}},_updateUser:function(){this.current_user=Galaxy.currUser.get("id");this.current_history=null;if(this.current_user){this.current_history=Galaxy.currHistoryPanel.model.get("id")}},_updateScreen:function(){this._updateUser();if(this.counter.announce==0){if(this.uploadbox.compatible()){message="You can Drag & Drop files into this box."}else{message="Unfortunately, your browser does not support multiple file uploads or drag&drop.<br>Some supported browsers are: Firefox 4+, Chrome 7+, IE 10+, Opera 12+ or Safari 6+."}}else{if(this.counter.running==0){message="You added "+this.counter.announce+" file(s) to the queue. Add more files or click 'Start' to proceed."}else{message="Please wait..."+this.counter.announce+" out of "+this.counter.running+" remaining."}}$("#upload-info").html(message);if(this.counter.running==0&&this.counter.announce+this.counter.success+this.counter.error>0){this.modal.enableButton("Reset")}else{this.modal.disableButton("Reset")}if(this.counter.running==0&&this.counter.announce>0){this.modal.enableButton("Start")}else{this.modal.disableButton("Start")}if(this.counter.running>0){this.modal.enableButton("Pause")}else{this.modal.disableButton("Pause")}if(this.counter.running==0){this.modal.enableButton("Choose local file");this.modal.enableButton("Choose FTP file");this.modal.enableButton("Create new file")}else{this.modal.disableButton("Choose local file");this.modal.disableButton("Choose FTP file");this.modal.disableButton("Create new file")}if(this.current_user&&this.options.ftp_upload_dir&&this.options.ftp_upload_site){this.modal.showButton("Choose FTP file")}else{this.modal.hideButton("Choose FTP file")}if(this.counter.announce+this.counter.success+this.counter.error>0){$(this.el).find("#upload-table").show()}else{$(this.el).find("#upload-table").hide()}},_upload_percentage:function(h,i){return(this.upload_completed+(h*i))/this.upload_size},_template:function(i,h){return'<div id="'+i+'" class="upload-box"><table id="upload-table" class="table table-striped" style="display: none;"><thead><tr><th>Name</th><th>Size</th><th>Type</th><th>Genome</th><th>Settings</th><th>Status</th><th></th></tr></thead><tbody></tbody></table></div><h6 id="'+h+'" class="upload-info"></h6>'}})}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for deleting uploading HDAs.
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/05c93f3b424c/ Changeset: 05c93f3b424c User: dan Date: 2014-02-18 18:57:39 Summary: Fix for deleting uploading HDAs. Affected #: 1 file diff -r 5c3d7c2346617774a4248b9ef990a3208e00576b -r 05c93f3b424c372c14468f9aa25a79db88bfca24 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -274,7 +274,9 @@ return { 'error': 'Anonymous users cannot edit datasets outside their current history' } else: payload = self._validate_and_parse_update_payload( payload ) - hda = self.get_dataset( trans, id, check_ownership=True, check_accessible=True, check_state=True ) + # only check_state if not deleting, otherwise cannot delete uploading files + check_state = not payload.get( 'deleted', False ) + hda = self.get_dataset( trans, id, check_ownership=True, check_accessible=True, check_state=check_state ) # get_dataset can return a string during an error if hda and isinstance( hda, trans.model.HistoryDatasetAssociation ): changed = self.set_hda_from_dict( trans, hda, payload ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/05b757b20e06/ Changeset: 05b757b20e06 User: jmchilton Date: 2014-02-18 18:33:25 Summary: Random PEP-8 fixes to some api controllers. Affected #: 3 files diff -r a04725d3afedb65329b13bf5ba813e6381ec082f -r 05b757b20e06ee2667a3e910cf24e64f14915953 lib/galaxy/webapps/galaxy/api/libraries.py --- a/lib/galaxy/webapps/galaxy/api/libraries.py +++ b/lib/galaxy/webapps/galaxy/api/libraries.py @@ -10,6 +10,7 @@ import logging log = logging.getLogger( __name__ ) + class LibrariesController( BaseAPIController ): @web.expose_api @@ -130,7 +131,7 @@ new_library['synopsis'] = synopsis new_library['id'] = encoded_id return new_library - + def edit( self, trans, encoded_id, payload, **kwd ): """ * PUT /api/libraries/{encoded_id} diff -r a04725d3afedb65329b13bf5ba813e6381ec082f -r 05b757b20e06ee2667a3e910cf24e64f14915953 lib/galaxy/webapps/galaxy/api/library_contents.py --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -2,7 +2,7 @@ API operations on the contents of a library. """ import logging -from galaxy import web , exceptions +from galaxy import web, exceptions from galaxy.model import ExtendedMetadata, ExtendedMetadataIndex from galaxy.web.base.controller import BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin @@ -11,6 +11,7 @@ log = logging.getLogger( __name__ ) + class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin ): @@ -36,6 +37,7 @@ """ rval = [] current_user_roles = trans.get_current_user_roles() + def traverse( folder ): admin = trans.user_is_admin() rval = [] @@ -50,7 +52,9 @@ for ld in folder.datasets: if not admin: can_access = trans.app.security_agent.can_access_dataset( - current_user_roles, ld.library_dataset_dataset_association.dataset ) + current_user_roles, + ld.library_dataset_dataset_association.dataset + ) if (admin or can_access) and not ld.deleted: #log.debug( "type(folder): %s" % type( folder ) ) #log.debug( "type(api_path): %s; folder.api_path: %s" % ( type(folder.api_path), folder.api_path ) ) @@ -73,20 +77,20 @@ return "Invalid library id ( %s ) specified." % str( library_id ) #log.debug( "Root folder type: %s" % type( library.root_folder ) ) encoded_id = 'F' + trans.security.encode_id( library.root_folder.id ) - rval.append( dict( id = encoded_id, - type = 'folder', - name = '/', - url = url_for( 'library_content', library_id=library_id, id=encoded_id ) ) ) + rval.append( dict( id=encoded_id, + type='folder', + name='/', + url=url_for( 'library_content', library_id=library_id, id=encoded_id ) ) ) #log.debug( "Root folder attributes: %s" % str(dir(library.root_folder)) ) library.root_folder.api_path = '' for content in traverse( library.root_folder ): encoded_id = trans.security.encode_id( content.id ) if content.api_type == 'folder': encoded_id = 'F' + encoded_id - rval.append( dict( id = encoded_id, - type = content.api_type, - name = content.api_path, - url = url_for( 'library_content', library_id=library_id, id=encoded_id, ) ) ) + rval.append( dict( id=encoded_id, + type=content.api_type, + name=content.api_path, + url=url_for( 'library_content', library_id=library_id, id=encoded_id, ) ) ) return rval @web.expose_api diff -r a04725d3afedb65329b13bf5ba813e6381ec082f -r 05b757b20e06ee2667a3e910cf24e64f14915953 lib/galaxy/webapps/galaxy/api/roles.py --- a/lib/galaxy/webapps/galaxy/api/roles.py +++ b/lib/galaxy/webapps/galaxy/api/roles.py @@ -4,11 +4,12 @@ import logging from galaxy.web.base.controller import BaseAPIController, url_for from galaxy import web -from elementtree.ElementTree import XML log = logging.getLogger( __name__ ) + class RoleAPIController( BaseAPIController ): + @web.expose_api def index( self, trans, **kwd ): """ @@ -61,11 +62,11 @@ if not name or not description: trans.response.status = 400 return "Enter a valid name and a description" - if trans.sa_session.query( trans.app.model.Role ).filter( trans.app.model.Role.table.c.name==name ).first(): + if trans.sa_session.query( trans.app.model.Role ).filter( trans.app.model.Role.table.c.name == name ).first(): trans.response.status = 400 return "A role with that name already exists" - role_type = trans.app.model.Role.types.ADMIN #TODO: allow non-admins to create roles + role_type = trans.app.model.Role.types.ADMIN # TODO: allow non-admins to create roles role = trans.app.model.Role( name=name, description=description, type=role_type ) trans.sa_session.add( role ) @@ -73,12 +74,15 @@ users = [ trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( i ) ) for i in user_ids ] group_ids = payload.get( 'group_ids', [] ) groups = [ trans.sa_session.query( trans.model.Group ).get( trans.security.decode_id( i ) ) for i in group_ids ] + # Create the UserRoleAssociations for user in users: trans.app.security_agent.associate_user_role( user, role ) + # Create the GroupRoleAssociations for group in groups: trans.app.security_agent.associate_group_role( group, role ) + trans.sa_session.flush() encoded_id = trans.security.encode_id( role.id ) item = role.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id } ) https://bitbucket.org/galaxy/galaxy-central/commits/98253ff31a33/ Changeset: 98253ff31a33 User: jmchilton Date: 2014-02-18 18:33:28 Summary: Add note to job_conf.xml.sample_advanced about LWR+MQ. Affected #: 1 file diff -r 05b757b20e06ee2667a3e910cf24e64f14915953 -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -23,6 +23,11 @@  +  +  + </plugin><plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" /><plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" /> https://bitbucket.org/galaxy/galaxy-central/commits/5c3d7c234661/ Changeset: 5c3d7c234661 User: jmchilton Date: 2014-02-18 18:33:28 Summary: Improvement to message queue driven LWR. At high-level there are two core enhancements here - ability to kill message queue driven LWR jobs (though due to regressions in latest galaxy release there is not longer a way to initiate this from the GUI I don't think) and additional state transition (LWR+MQ jobs will now transition from queue to running properly, previously they just went from queue to complete). A lot of other small changes to LWR client aimed at getting file size of lwr_client/client.py and lwr_client/manager.py under control - see LWR commit log for more details. This updates the LWR client through LWR changeset 2d9f333. Affected #: 9 files diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -52,7 +52,7 @@ try: client = self.get_client_from_state(job_state) - if not hasattr(client, 'get_status'): + if hasattr(self.client_manager, 'ensure_has_status_update_callback'): # Message queue implementation. # TODO: Very hacky now, refactor after Dannon merges in his @@ -60,7 +60,7 @@ # check_watched_item like this and instead a callback needs to # be issued post job recovery allowing a message queue # consumer to be setup. - self.client_manager.ensure_has_job_completes_callback(self.__async_complete) + self.client_manager.ensure_has_status_update_callback(self.__async_update) return job_state status = client.get_status() @@ -69,16 +69,20 @@ # either way we are done I guess. self.mark_as_finished(job_state) return None - if status == "complete": + job_state = self.__update_job_state_for_lwr_status(job_state, status) + return job_state + + def __update_job_state_for_lwr_status(self, job_state, lwr_status): + if lwr_status == "complete": self.mark_as_finished(job_state) return None - if status == "running" and not job_state.running: + if lwr_status == "running" and not job_state.running: job_state.running = True job_state.job_wrapper.change_state( model.Job.states.RUNNING ) return job_state - def __async_complete( self, final_status ): - job_id = final_status[ "job_id" ] + def __async_update( self, full_status ): + job_id = full_status[ "job_id" ] job_state = self.__find_watched_job( job_id ) if not job_state: # Probably finished too quickly, sleep and try again. @@ -88,9 +92,9 @@ sleep( 2 ) job_state = self.__find_watched_job( job_id ) if not job_state: - log.warn( "Failed to find job corresponding to final status %s in %s" % ( final_status, self.watched ) ) + log.warn( "Failed to find job corresponding to final status %s in %s" % ( full_status, self.watched ) ) else: - self.mark_as_finished( job_state ) + self.__update_job_state_for_lwr_status(job_state, full_status["status"]) def __find_watched_job( self, job_id ): found_job = None @@ -227,7 +231,7 @@ job_wrapper = job_state.job_wrapper try: client = self.get_client_from_state(job_state) - run_results = client.final_status() + run_results = client.full_status() stdout = run_results.get('stdout', '') stderr = run_results.get('stderr', '') @@ -251,7 +255,7 @@ except Exception: message = "Failed to communicate with remote job server." job_wrapper.fail( message, exception=True ) - log.exception("failure running job %d" % job_wrapper.job_id) + log.exception("failure finishing job %d" % job_wrapper.job_id) return if not LwrJobRunner.__remote_metadata( client ): self._handle_metadata_externally( job_wrapper, resolve_requirements=True ) diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py --- a/lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py +++ b/lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py @@ -40,10 +40,15 @@ self.__exchange = kombu.Exchange(DEFAULT_EXCHANGE_NAME, DEFAULT_EXCHANGE_TYPE) self.__timeout = timeout + @property + def url(self): + return self.__url + def consume(self, queue_name, callback, check=True, connection_kwargs={}): queue = self.__queue(queue_name) with self.connection(self.__url, **connection_kwargs) as connection: with kombu.Consumer(connection, queues=[queue], callbacks=[callback], accept=['json']): + log.debug("Consuming queue %s" % queue) while check: try: connection.drain_events(timeout=self.__timeout) @@ -54,6 +59,7 @@ with self.connection(self.__url) as connection: with pools.producers[connection].acquire() as producer: key = self.__queue_name(name) + log.debug("Publishing with key %s and payload %s" % (key, payload)) producer.publish( payload, serializer='json', diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -1,49 +1,18 @@ import os -import shutil -import json from json import dumps -from time import sleep from .destination import submit_params from .setup_handler import build as build_setup_handler from .job_directory import RemoteJobDirectory +from .decorators import parseJson +from .decorators import retry +from .util import copy +from .util import ensure_directory import logging log = logging.getLogger(__name__) CACHE_WAIT_SECONDS = 3 -MAX_RETRY_COUNT = 5 -RETRY_SLEEP_TIME = 0.1 - - -class parseJson(object): - - def __call__(self, func): - def replacement(*args, **kwargs): - response = func(*args, **kwargs) - return json.loads(response) - return replacement - - -class retry(object): - - def __call__(self, func): - - def replacement(*args, **kwargs): - max_count = MAX_RETRY_COUNT - count = 0 - while True: - count += 1 - try: - return func(*args, **kwargs) - except: - if count >= max_count: - raise - else: - sleep(RETRY_SLEEP_TIME) - continue - - return replacement class OutputNotFoundException(Exception): @@ -111,17 +80,78 @@ super(JobClient, self).__init__(destination_params, job_id) self.job_manager_interface = job_manager_interface - def _raw_execute(self, command, args={}, data=None, input_path=None, output_path=None): - return self.job_manager_interface.execute(command, args, data, input_path, output_path) + def launch(self, command_line, requirements=[], remote_staging=[], job_config=None): + """ + Queue up the execution of the supplied `command_line` on the remote + server. Called launch for historical reasons, should be renamed to + enqueue or something like that. - @property - def _submit_params(self): - return submit_params(self.destination_params) + **Parameters** + + command_line : str + Command to execute. + """ + launch_params = dict(command_line=command_line, job_id=self.job_id) + submit_params_dict = submit_params(self.destination_params) + if submit_params_dict: + launch_params['params'] = dumps(submit_params_dict) + if requirements: + launch_params['requirements'] = dumps([requirement.to_dict() for requirement in requirements]) + if remote_staging: + launch_params['remote_staging'] = dumps(remote_staging) + if job_config and self.setup_handler.local: + # Setup not yet called, job properties were inferred from + # destination arguments. Hence, must have LWR setup job + # before queueing. + setup_params = _setup_params_from_job_config(job_config) + launch_params["setup_params"] = dumps(setup_params) + return self._raw_execute("launch", launch_params) + + def full_status(self): + """ Return a dictionary summarizing final state of job. + """ + return self.raw_check_complete() + + def kill(self): + """ + Cancel remote job, either removing from the queue or killing it. + """ + return self._raw_execute("kill", {"job_id": self.job_id}) + + @retry() + @parseJson() + def raw_check_complete(self): + """ + Get check_complete response from the remote server. + """ + check_complete_response = self._raw_execute("check_complete", {"job_id": self.job_id}) + return check_complete_response + + def get_status(self): + check_complete_response = self.raw_check_complete() + # Older LWR instances won't set status so use 'complete', at some + # point drop backward compatibility. + status = check_complete_response.get("status", None) + if status in ["status", None]: + # LEGACY: Bug in certains older LWR instances returned literal + # "status". + complete = check_complete_response["complete"] == "true" + old_status = "complete" if complete else "running" + status = old_status + return status + + def clean(self): + """ + Cleanup the remote job. + """ + self._raw_execute("clean", {"job_id": self.job_id}) @parseJson() - def input_path(self, path, input_type, name=None): - args = {"job_id": self.job_id, "name": name, "input_type": input_type} - return self._raw_execute('input_path', args) + def remote_setup(self, **setup_args): + """ + Setup remote LWR server to run this job. + """ + return self._raw_execute("setup", setup_args) def put_file(self, path, input_type, name=None, contents=None, action_type='transfer'): if not name: @@ -134,9 +164,80 @@ return self._upload_file(args, contents, input_path) elif action_type == 'copy': lwr_path = self._raw_execute('input_path', args) - self._copy(path, lwr_path) + copy(path, lwr_path) return {'path': lwr_path} + def fetch_output(self, path, name, working_directory, action_type, output_type): + """ + Fetch (transfer, copy, etc...) an output from the remote LWR server. + + **Parameters** + + path : str + Local path of the dataset. + name : str + Remote name of file (i.e. path relative to remote staging output + or working directory). + working_directory : str + Local working_directory for the job. + action_type : str + Where to find file on LWR (output_workdir or output). legacy is also + an option in this case LWR is asked for location - this will only be + used if targetting an older LWR server that didn't return statuses + allowing this to be inferred. + """ + if output_type == 'legacy': + self._fetch_output_legacy(path, working_directory, action_type=action_type) + elif output_type == 'output_workdir': + self._fetch_work_dir_output(name, working_directory, path, action_type=action_type) + elif output_type == 'output': + self._fetch_output(path=path, name=name, action_type=action_type) + else: + raise Exception("Unknown output_type %s" % output_type) + + def _raw_execute(self, command, args={}, data=None, input_path=None, output_path=None): + return self.job_manager_interface.execute(command, args, data, input_path, output_path) + + # Deprecated + def _fetch_output_legacy(self, path, working_directory, action_type='transfer'): + # Needs to determine if output is task/working directory or standard. + name = os.path.basename(path) + + output_type = self._get_output_type(name) + if output_type == "none": + # Just make sure the file was created. + if not os.path.exists(path): + raise OutputNotFoundException(path) + return + elif output_type in ["task"]: + path = os.path.join(working_directory, name) + + self.__populate_output_path(name, path, output_type, action_type) + + def _fetch_output(self, path, name=None, check_exists_remotely=False, action_type='transfer'): + if not name: + # Extra files will send in the path. + name = os.path.basename(path) + + output_type = "direct" # Task/from_work_dir outputs now handled with fetch_work_dir_output + self.__populate_output_path(name, path, output_type, action_type) + + def _fetch_work_dir_output(self, name, working_directory, output_path, action_type='transfer'): + ensure_directory(output_path) + if action_type == 'transfer': + self.__raw_download_output(name, self.job_id, "work_dir", output_path) + else: # Even if action is none - LWR has a different work_dir so this needs to be copied. + lwr_path = self._output_path(name, self.job_id, 'work_dir')['path'] + copy(lwr_path, output_path) + + def __populate_output_path(self, name, output_path, output_type, action_type): + ensure_directory(output_path) + if action_type == 'transfer': + self.__raw_download_output(name, self.job_id, output_type, output_path) + elif action_type == 'copy': + lwr_path = self._output_path(name, self.job_id, output_type)['path'] + copy(lwr_path, output_path) + @parseJson() def _upload_file(self, args, contents, input_path): return self._raw_execute(self._upload_file_action(args), args, contents, input_path) @@ -163,75 +264,6 @@ return self._raw_execute("get_output_type", {"name": name, "job_id": self.job_id}) - # Deprecated - def fetch_output_legacy(self, path, working_directory, action_type='transfer'): - # Needs to determine if output is task/working directory or standard. - name = os.path.basename(path) - - output_type = self._get_output_type(name) - if output_type == "none": - # Just make sure the file was created. - if not os.path.exists(path): - raise OutputNotFoundException(path) - return - elif output_type in ["task"]: - path = os.path.join(working_directory, name) - - self.__populate_output_path(name, path, output_type, action_type) - - def fetch_output(self, path, name=None, check_exists_remotely=False, action_type='transfer'): - """ - Download an output dataset from the remote server. - - **Parameters** - - path : str - Local path of the dataset. - working_directory : str - Local working_directory for the job. - """ - - if not name: - # Extra files will send in the path. - name = os.path.basename(path) - - output_type = "direct" # Task/from_work_dir outputs now handled with fetch_work_dir_output - self.__populate_output_path(name, path, output_type, action_type) - - def __populate_output_path(self, name, output_path, output_type, action_type): - self.__ensure_directory(output_path) - if action_type == 'transfer': - self.__raw_download_output(name, self.job_id, output_type, output_path) - elif action_type == 'copy': - lwr_path = self._output_path(name, self.job_id, output_type)['path'] - self._copy(lwr_path, output_path) - - def fetch_work_dir_output(self, name, working_directory, output_path, action_type='transfer'): - """ - Download an output dataset specified with from_work_dir from the - remote server. - - **Parameters** - - name : str - Path in job's working_directory to find output in. - working_directory : str - Local working_directory for the job. - output_path : str - Full path to output dataset. - """ - self.__ensure_directory(output_path) - if action_type == 'transfer': - self.__raw_download_output(name, self.job_id, "work_dir", output_path) - else: # Even if action is none - LWR has a different work_dir so this needs to be copied. - lwr_path = self._output_path(name, self.job_id, 'work_dir')['path'] - self._copy(lwr_path, output_path) - - def __ensure_directory(self, output_path): - output_path_directory = os.path.dirname(output_path) - if not os.path.exists(output_path_directory): - os.makedirs(output_path_directory) - @parseJson() def _output_path(self, name, job_id, output_type): return self._raw_execute("output_path", @@ -248,107 +280,6 @@ } self._raw_execute("download_output", output_params, output_path=output_path) - def launch(self, command_line, requirements=[], remote_staging=[], job_config=None): - """ - Queue up the execution of the supplied `command_line` on the remote - server. Called launch for historical reasons, should be renamed to - enqueue or something like that. - - **Parameters** - - command_line : str - Command to execute. - """ - launch_params = dict(command_line=command_line, job_id=self.job_id) - submit_params = self._submit_params - if submit_params: - launch_params['params'] = dumps(submit_params) - if requirements: - launch_params['requirements'] = dumps([requirement.to_dict() for requirement in requirements]) - if remote_staging: - launch_params['remote_staging'] = dumps(remote_staging) - if job_config and self.setup_handler.local: - # Setup not yet called, job properties were inferred from - # destination arguments. Hence, must have LWR setup job - # before queueing. - setup_params = _setup_params_from_job_config(job_config) - launch_params["setup_params"] = dumps(setup_params) - return self._raw_execute("launch", launch_params) - - def final_status(self): - """ Return a dictionary summarizing final state of job. - """ - return self.raw_check_complete() - - def kill(self): - """ - Cancel remote job, either removing from the queue or killing it. - """ - return self._raw_execute("kill", {"job_id": self.job_id}) - - def wait(self, max_seconds=None): - """ - Wait for job to finish. - """ - i = 0 - while max_seconds is None or i < max_seconds: - complete_response = self.raw_check_complete() - if complete_response["complete"] == "true": - print complete_response - return complete_response - else: - print complete_response - sleep(1) - i += 1 - - @parseJson() - def raw_check_complete(self): - """ - Get check_complete response from the remote server. - """ - check_complete_response = self._raw_execute("check_complete", {"job_id": self.job_id}) - return check_complete_response - - def check_complete(self, response=None): - """ - Return boolean indicating whether the job is complete. - """ - if response is None: - response = self.raw_check_complete() - return response["complete"] == "true" - - @retry() - def get_status(self): - check_complete_response = self.raw_check_complete() - # Older LWR instances won't set status so use 'complete', at some - # point drop backward compatibility. - status = check_complete_response.get("status", None) - # Bug in certains older LWR instances returned literal "status". - if status in ["status", None]: - complete = self.check_complete(check_complete_response) - old_status = "complete" if complete else "running" - status = old_status - return status - - def clean(self): - """ - Cleanup the remote job. - """ - self._raw_execute("clean", {"job_id": self.job_id}) - - @parseJson() - def remote_setup(self, **setup_args): - """ - Setup remote LWR server to run this job. - """ - return self._raw_execute("setup", setup_args) - - def _copy(self, source, destination): - source = os.path.abspath(source) - destination = os.path.abspath(destination) - if source != destination: - shutil.copyfile(source, destination) - class MessageJobClient(BaseJobClient): @@ -379,16 +310,16 @@ return self.client_manager.exchange.publish("setup", launch_params) def clean(self): - del self.client_manager.final_status_cache[self.job_id] + del self.client_manager.status_cache[self.job_id] - def final_status(self): - final_status = self.client_manager.final_status_cache.get(self.job_id, None) - if final_status is None: - raise Exception("final_status() called before a final status was properly cached with cilent manager.") - return final_status + def full_status(self): + full_status = self.client_manager.status_cache.get(self.job_id, None) + if full_status is None: + raise Exception("full_status() called before a final status was properly cached with cilent manager.") + return full_status def kill(self): - log.warn("Kill not yet implemented with message queue driven LWR jobs.") + self.client_manager.exchange.publish("kill", dict(job_id=self.job_id)) class InputCachingJobClient(JobClient): @@ -443,55 +374,3 @@ tool_id=tool_id, tool_version=tool_version ) - - -class ObjectStoreClient(object): - - def __init__(self, lwr_interface): - self.lwr_interface = lwr_interface - - @parseJson() - def exists(self, **kwds): - return self._raw_execute("object_store_exists", args=self.__data(**kwds)) - - @parseJson() - def file_ready(self, **kwds): - return self._raw_execute("object_store_file_ready", args=self.__data(**kwds)) - - @parseJson() - def create(self, **kwds): - return self._raw_execute("object_store_create", args=self.__data(**kwds)) - - @parseJson() - def empty(self, **kwds): - return self._raw_execute("object_store_empty", args=self.__data(**kwds)) - - @parseJson() - def size(self, **kwds): - return self._raw_execute("object_store_size", args=self.__data(**kwds)) - - @parseJson() - def delete(self, **kwds): - return self._raw_execute("object_store_delete", args=self.__data(**kwds)) - - @parseJson() - def get_data(self, **kwds): - return self._raw_execute("object_store_get_data", args=self.__data(**kwds)) - - @parseJson() - def get_filename(self, **kwds): - return self._raw_execute("object_store_get_filename", args=self.__data(**kwds)) - - @parseJson() - def update_from_file(self, **kwds): - return self._raw_execute("object_store_update_from_file", args=self.__data(**kwds)) - - @parseJson() - def get_store_usage_percent(self): - return self._raw_execute("object_store_get_store_usage_percent", args={}) - - def __data(self, **kwds): - return kwds - - def _raw_execute(self, command, args={}): - return self.lwr_interface.execute(command, args, data=None, input_path=None, output_path=None) diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr_client/decorators.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/decorators.py @@ -0,0 +1,35 @@ +import time +import json + +MAX_RETRY_COUNT = 5 +RETRY_SLEEP_TIME = 0.1 + + +class parseJson(object): + + def __call__(self, func): + def replacement(*args, **kwargs): + response = func(*args, **kwargs) + return json.loads(response) + return replacement + + +class retry(object): + + def __call__(self, func): + + def replacement(*args, **kwargs): + max_count = MAX_RETRY_COUNT + count = 0 + while True: + count += 1 + try: + return func(*args, **kwargs) + except: + if count >= max_count: + raise + else: + time.sleep(RETRY_SLEEP_TIME) + continue + + return replacement diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr_client/interface.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/interface.py @@ -0,0 +1,97 @@ +from abc import ABCMeta +from abc import abstractmethod +try: + from StringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO +try: + from six import text_type +except ImportError: + from galaxy.util import unicodify as text_type +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode + + +class LwrInteface(object): + """ + Abstract base class describes how synchronous client communicates with + (potentially remote) LWR procedures. Obvious implementation is HTTP based + but LWR objects wrapped in routes can also be directly communicated with + if in memory. + """ + __metaclass__ = ABCMeta + + @abstractmethod + def execute(self, command, args={}, data=None, input_path=None, output_path=None): + """ + Execute the correspond command against configured LWR job manager. Arguments are + method parameters and data or input_path describe essentially POST bodies. If command + results in a file, resulting path should be specified as output_path. + """ + + +class HttpLwrInterface(LwrInteface): + + def __init__(self, destination_params, transport): + self.transport = transport + self.remote_host = destination_params.get("url") + assert self.remote_host is not None, "Failed to determine url for LWR client." + self.private_key = destination_params.get("private_token", None) + + def execute(self, command, args={}, data=None, input_path=None, output_path=None): + url = self.__build_url(command, args) + response = self.transport.execute(url, data=data, input_path=input_path, output_path=output_path) + return response + + def __build_url(self, command, args): + if self.private_key: + args["private_key"] = self.private_key + arg_bytes = dict([(k, text_type(args[k]).encode('utf-8')) for k in args]) + data = urlencode(arg_bytes) + url = self.remote_host + command + "?" + data + return url + + +class LocalLwrInterface(LwrInteface): + + def __init__(self, destination_params, job_manager=None, file_cache=None, object_store=None): + self.job_manager = job_manager + self.file_cache = file_cache + self.object_store = object_store + + def __app_args(self): + ## Arguments that would be specified from LwrApp if running + ## in web server. + return { + 'manager': self.job_manager, + 'file_cache': self.file_cache, + 'object_store': self.object_store, + 'ip': None + } + + def execute(self, command, args={}, data=None, input_path=None, output_path=None): + # If data set, should be unicode (on Python 2) or str (on Python 3). + from lwr import routes + from lwr.framework import build_func_args + controller = getattr(routes, command) + action = controller.func + body_args = dict(body=self.__build_body(data, input_path)) + args = build_func_args(action, args.copy(), self.__app_args(), body_args) + result = action(**args) + if controller.response_type != 'file': + return controller.body(result) + else: + # TODO: Add to Galaxy. + from galaxy.util import copy_to_path + with open(result, 'rb') as result_file: + copy_to_path(result_file, output_path) + + def __build_body(self, data, input_path): + if data is not None: + return BytesIO(data.encode('utf-8')) + elif input_path is not None: + return open(input_path, 'rb') + else: + return None diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr_client/manager.py --- a/lib/galaxy/jobs/runners/lwr_client/manager.py +++ b/lib/galaxy/jobs/runners/lwr_client/manager.py @@ -1,27 +1,16 @@ -from abc import ABCMeta, abstractmethod import threading try: from Queue import Queue except ImportError: from queue import Queue from os import getenv -try: - from urllib import urlencode -except ImportError: - from urllib.parse import urlencode -try: - from StringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO -try: - from six import text_type -except ImportError: - from galaxy.util import unicodify as text_type from .client import JobClient from .client import InputCachingJobClient -from .client import ObjectStoreClient from .client import MessageJobClient +from .interface import HttpLwrInterface +from .interface import LocalLwrInterface +from .object_client import ObjectStoreClient from .transport import get_transport from .util import TransferEventManager from .destination import url_to_destination_params @@ -87,12 +76,12 @@ self.url = kwds.get('url') self.manager_name = kwds.get("manager", "_default_") self.exchange = LwrExchange(self.url, self.manager_name) - self.final_status_cache = {} + self.status_cache = {} self.callback_lock = threading.Lock() self.callback_thread = None self.active = True - def ensure_has_job_completes_callback(self, callback): + def ensure_has_status_update_callback(self, callback): with self.callback_lock: if self.callback_thread is not None: return @@ -100,17 +89,17 @@ def callback_wrapper(body, message): try: if "job_id" in body: - self.final_status_cache[body["job_id"]] = body + self.status_cache[body["job_id"]] = body callback(body) except Exception: log.exception("Failure processing job status update message.") message.ack() def run(): - self.exchange.consume("complete", callback_wrapper, check=self) + self.exchange.consume("status_update", callback_wrapper, check=self) thread = threading.Thread( - name="lwr_client_%s_complete_callback" % self.manager_name, + name="lwr_client_%s_status_update_callback" % self.manager_name, target=run ) thread.daemon = False # Lets not interrupt processing of this. @@ -148,87 +137,6 @@ return ObjectStoreClient(interface) -class JobManagerInteface(object): - """ - Abstract base class describes how client communicates with remote job - manager. - """ - __metaclass__ = ABCMeta - - @abstractmethod - def execute(self, command, args={}, data=None, input_path=None, output_path=None): - """ - Execute the correspond command against configured LWR job manager. Arguments are - method parameters and data or input_path describe essentially POST bodies. If command - results in a file, resulting path should be specified as output_path. - """ - - -class HttpLwrInterface(object): - - def __init__(self, destination_params, transport): - self.transport = transport - self.remote_host = destination_params.get("url") - assert self.remote_host is not None, "Failed to determine url for LWR client." - self.private_key = destination_params.get("private_token", None) - - def execute(self, command, args={}, data=None, input_path=None, output_path=None): - url = self.__build_url(command, args) - response = self.transport.execute(url, data=data, input_path=input_path, output_path=output_path) - return response - - def __build_url(self, command, args): - if self.private_key: - args["private_key"] = self.private_key - arg_bytes = dict([(k, text_type(args[k]).encode('utf-8')) for k in args]) - data = urlencode(arg_bytes) - url = self.remote_host + command + "?" + data - return url - - -class LocalLwrInterface(object): - - def __init__(self, destination_params, job_manager=None, file_cache=None, object_store=None): - self.job_manager = job_manager - self.file_cache = file_cache - self.object_store = object_store - - def __app_args(self): - ## Arguments that would be specified from LwrApp if running - ## in web server. - return { - 'manager': self.job_manager, - 'file_cache': self.file_cache, - 'object_store': self.object_store, - 'ip': None - } - - def execute(self, command, args={}, data=None, input_path=None, output_path=None): - # If data set, should be unicode (on Python 2) or str (on Python 3). - from lwr import routes - from lwr.framework import build_func_args - controller = getattr(routes, command) - action = controller.func - body_args = dict(body=self.__build_body(data, input_path)) - args = build_func_args(action, args.copy(), self.__app_args(), body_args) - result = action(**args) - if controller.response_type != 'file': - return controller.body(result) - else: - # TODO: Add to Galaxy. - from galaxy.util import copy_to_path - with open(result, 'rb') as result_file: - copy_to_path(result_file, output_path) - - def __build_body(self, data, input_path): - if data is not None: - return BytesIO(data.encode('utf-8')) - elif input_path is not None: - return open(input_path, 'rb') - else: - return None - - class ClientCacher(object): def __init__(self, **kwds): diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr_client/object_client.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/object_client.py @@ -0,0 +1,53 @@ +from .decorators import parseJson + + +class ObjectStoreClient(object): + + def __init__(self, lwr_interface): + self.lwr_interface = lwr_interface + + @parseJson() + def exists(self, **kwds): + return self._raw_execute("object_store_exists", args=self.__data(**kwds)) + + @parseJson() + def file_ready(self, **kwds): + return self._raw_execute("object_store_file_ready", args=self.__data(**kwds)) + + @parseJson() + def create(self, **kwds): + return self._raw_execute("object_store_create", args=self.__data(**kwds)) + + @parseJson() + def empty(self, **kwds): + return self._raw_execute("object_store_empty", args=self.__data(**kwds)) + + @parseJson() + def size(self, **kwds): + return self._raw_execute("object_store_size", args=self.__data(**kwds)) + + @parseJson() + def delete(self, **kwds): + return self._raw_execute("object_store_delete", args=self.__data(**kwds)) + + @parseJson() + def get_data(self, **kwds): + return self._raw_execute("object_store_get_data", args=self.__data(**kwds)) + + @parseJson() + def get_filename(self, **kwds): + return self._raw_execute("object_store_get_filename", args=self.__data(**kwds)) + + @parseJson() + def update_from_file(self, **kwds): + return self._raw_execute("object_store_update_from_file", args=self.__data(**kwds)) + + @parseJson() + def get_store_usage_percent(self): + return self._raw_execute("object_store_get_store_usage_percent", args={}) + + def __data(self, **kwds): + return kwds + + def _raw_execute(self, command, args={}): + return self.lwr_interface.execute(command, args, data=None, input_path=None, output_path=None) diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr_client/staging/down.py --- a/lib/galaxy/jobs/runners/lwr_client/staging/down.py +++ b/lib/galaxy/jobs/runners/lwr_client/staging/down.py @@ -41,15 +41,14 @@ if not action.staging_action_local: return False - path = action.path - if output_type == 'legacy': - working_directory = results_collector.client_outputs.working_directory - self.client.fetch_output_legacy(path, working_directory, action_type=action.action_type) - elif output_type == 'output_workdir': - working_directory = results_collector.client_outputs.working_directory - self.client.fetch_work_dir_output(name, working_directory, path, action_type=action.action_type) - elif output_type == 'output': - self.client.fetch_output(path=path, name=name, action_type=action.action_type) + working_directory = results_collector.client_outputs.working_directory + self.client.fetch_output( + path=action.path, + name=name, + working_directory=working_directory, + output_type=output_type, + action_type=action.action_type + ) return True diff -r 98253ff31a334d01eb2ee28e10c637f71c1ba6f3 -r 5c3d7c2346617774a4248b9ef990a3208e00576b lib/galaxy/jobs/runners/lwr_client/util.py --- a/lib/galaxy/jobs/runners/lwr_client/util.py +++ b/lib/galaxy/jobs/runners/lwr_client/util.py @@ -6,6 +6,7 @@ from os.path import join import os.path import hashlib +import shutil def unique_path_prefix(path): @@ -14,6 +15,22 @@ return m.hexdigest() +def copy(source, destination): + """ Copy file from source to destination if needed (skip if source + is destination). + """ + source = os.path.abspath(source) + destination = os.path.abspath(destination) + if source != destination: + shutil.copyfile(source, destination) + + +def ensure_directory(file_path): + directory = os.path.dirname(file_path) + if not os.path.exists(directory): + os.makedirs(directory) + + def directory_files(directory): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Tagging: prevent intialization error on tag autocomplete (http://stackoverflow.com/questions/21549126/error-cannot-call-methods-on-autocomplete-prior-to-initialization-attempted-to)
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a04725d3afed/ Changeset: a04725d3afed User: carlfeberhard Date: 2014-02-18 17:55:34 Summary: Tagging: prevent intialization error on tag autocomplete (http://stackoverflow.com/questions/21549126/error-cannot-call-methods-on-au…) Affected #: 1 file diff -r a9fb2779dd6948fb94d820dce89ccc70335e6c47 -r a04725d3afedb65329b13bf5ba813e6381ec082f templates/tagging_common.mako --- a/templates/tagging_common.mako +++ b/templates/tagging_common.mako @@ -211,7 +211,7 @@ use_toggle_link: ${iff( use_toggle_link, 'true', 'false' )} }; - $('#${elt_id}').autocomplete_tagging(options); + $('#${elt_id}').find( 'input' ).autocomplete_tagging(options); </script> ## Use style to hide/display the tag area. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Add kombu, amqp, anyjson eggs.
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a9fb2779dd69/ Changeset: a9fb2779dd69 User: dannon Date: 2014-02-18 17:47:23 Summary: Add kombu, amqp, anyjson eggs. Affected #: 1 file diff -r 3a5251e288011f02eff318a4dd88d76af4edcfd3 -r a9fb2779dd6948fb94d820dce89ccc70335e6c47 eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -32,7 +32,9 @@ ; msgpack_python = 0.2.4 [eggs:noplatform] +amqp = 1.4.3 amqplib = 0.6.1 +anyjson = 0.3.3 Beaker = 1.4 boto = 2.5.2 decorator = 3.1.2 @@ -41,6 +43,7 @@ elementtree = 1.2.6_20050316 Fabric = 1.7.0 GeneTrack = 2.0.0_beta_1 +kombu = 3.0.12 lrucache = 0.2 Mako = 0.4.1 nose = 0.11.1 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: HDA model: allow copying the purged attribute; History model: allow copying all datasets (including purged); History imp: change query param flag 'include_deleted' to 'all_datasets', send to History model copy; history/view: incorporate 'all_datasets' flag in import button
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3a5251e28801/ Changeset: 3a5251e28801 User: carlfeberhard Date: 2014-02-18 16:44:43 Summary: HDA model: allow copying the purged attribute; History model: allow copying all datasets (including purged); History imp: change query param flag 'include_deleted' to 'all_datasets', send to History model copy; history/view: incorporate 'all_datasets' flag in import button Affected #: 3 files diff -r 48f57ba4faed584929ab3ef31aa59f663d84a411 -r 3a5251e288011f02eff318a4dd88d76af4edcfd3 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -822,7 +822,12 @@ self.datasets.append( dataset ) return dataset - def copy( self, name=None, target_user=None, activatable=False ): + def copy( self, name=None, target_user=None, activatable=False, all_datasets=False ): + """ + Return a copy of this history using the given `name` and `target_user`. + If `activatable`, copy only non-deleted datasets. If `all_datasets`, copy + non-deleted, deleted, and purged datasets. + """ # Create new history. if not name: name = self.name @@ -845,6 +850,8 @@ # Copy HDAs. if activatable: hdas = self.activatable_datasets + elif all_datasets: + hdas = self.datasets else: hdas = self.active_datasets for hda in hdas: @@ -1682,6 +1689,9 @@ deleted=self.deleted, parent_id=parent_id, copied_from_history_dataset_association=self ) + # update init non-keywords as well + hda.purged = self.purged + object_session( self ).add( hda ) object_session( self ).flush() hda.set_size() diff -r 48f57ba4faed584929ab3ef31aa59f663d84a411 -r 3a5251e288011f02eff318a4dd88d76af4edcfd3 lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -783,8 +783,8 @@ else: referer_message = "<a href='%s'>go to Galaxy's start page</a>" % url_for( '/' ) - # include activatable/deleted datasets when copying? - include_deleted = util.string_as_bool( kwd.get( 'include_deleted', False ) ) + # include all datasets when copying? + all_datasets = util.string_as_bool( kwd.get( 'all_datasets', False ) ) # Do import. if not id: @@ -799,7 +799,7 @@ #dan: I can import my own history. #if import_history.user_id == user.id: # return trans.show_error_message( "You cannot import your own history.<br>You can %s." % referer_message, use_panels=True ) - new_history = import_history.copy( target_user=user, activatable=include_deleted ) + new_history = import_history.copy( target_user=user, all_datasets=all_datasets ) new_history.name = "imported: " + new_history.name new_history.user_id = user.id galaxy_session = trans.get_galaxy_session() diff -r 48f57ba4faed584929ab3ef31aa59f663d84a411 -r 3a5251e288011f02eff318a4dd88d76af4edcfd3 templates/webapps/galaxy/history/view.mako --- a/templates/webapps/galaxy/history/view.mako +++ b/templates/webapps/galaxy/history/view.mako @@ -98,7 +98,7 @@ %> %if not history[ 'purged' ]: <a id="import" class="btn btn-default" style="display: none;" - href="${h.url_for( controller='history', action='imp', id=history['id'], include_deleted=show_deleted )}"> + href="${h.url_for( controller='history', action='imp', id=history['id'], all_datasets=show_deleted )}"> ${_('Import and start using history')} </a> %endif @@ -133,7 +133,7 @@ // a bit hacky var href = $importBtn.attr( 'href' ), includeDeleted = $( this ).modeButton()[0].getMode().mode === 'exclude'; - href = href.replace( /include_deleted=True|False/, ( includeDeleted )?( 'True' ):( 'False' ) ); + href = href.replace( /all_datasets=True|False/, ( includeDeleted )?( 'True' ):( 'False' ) ); $importBtn.attr( 'href', href ); $importBtn.text( includeDeleted ? _l( 'Import with deleted datasets and start using history' ) : _l( 'Import and start using history' ) ); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix for recent LWR message queue commit.
by commits-noreply＠bitbucket.org 18 Feb '14

18 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/48f57ba4faed/ Changeset: 48f57ba4faed User: jmchilton Date: 2014-02-18 13:32:08 Summary: Fix for recent LWR message queue commit. Was always trying to bind to message queue. Affected #: 1 file diff -r 75b13d642f94b9ff6ec7c3c3713ad1177b243bd3 -r 48f57ba4faed584929ab3ef31aa59f663d84a411 lib/galaxy/jobs/runners/lwr_client/manager.py --- a/lib/galaxy/jobs/runners/lwr_client/manager.py +++ b/lib/galaxy/jobs/runners/lwr_client/manager.py @@ -37,7 +37,7 @@ def build_client_manager(**kwargs): if 'job_manager' in kwargs: return ClientManager(**kwargs) # TODO: Consider more separation here. - elif 'url' in kwargs: + elif kwargs.get('url', None): return MessageQueueClientManager(**kwargs) else: return ClientManager(**kwargs) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Allow LWR runner to publish/consume messages via message queue.
by commits-noreply＠bitbucket.org 17 Feb '14

17 Feb '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/75b13d642f94/ Changeset: 75b13d642f94 User: jmchilton Date: 2014-02-18 04:26:49 Summary: Allow LWR runner to publish/consume messages via message queue. To use LWR via message queue, ensure each such destination defines "default_file_action" as "remote_copy" and the LWR base staging directory pointed to by the "jobs_directory" parameter. Finally add a "url" argument to LWR job runner plugin in job_conf.xml specifing the amqp URL to target (by default this will target the default ("_default_") LWR job manager - this can be overridden by specifing a "manager" attribute the plugin element in job_conf.xml). Some of the implementation details here are bit hacky now to localize changes to LWR, once Dannon has merged his work - a new job runner base class or option should be implemented to account for this truely asynchronous job running implementation. (Comments inline about this.) This updates the LWR client through LWR changeset 59ad1ea03448. Affected #: 3 files diff -r 7d649885e3259f3359e8c3816315c60e958c1518 -r 75b13d642f94b9ff6ec7c3c3713ad1177b243bd3 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -35,12 +35,13 @@ """ runner_name = "LWRRunner" - def __init__( self, app, nworkers, transport=None, cache=None ): + def __init__( self, app, nworkers, transport=None, cache=None, url=None ): """Start the job runner """ super( LwrJobRunner, self ).__init__( app, nworkers ) + self.async_status_updates = dict() self._init_monitor_thread() self._init_worker_threads() - client_manager_kwargs = {'transport_type': transport, 'cache': string_as_bool_or_none(cache)} + client_manager_kwargs = {'transport_type': transport, 'cache': string_as_bool_or_none(cache), "url": url} self.client_manager = build_client_manager(**client_manager_kwargs) def url_to_destination( self, url ): @@ -50,6 +51,18 @@ def check_watched_item(self, job_state): try: client = self.get_client_from_state(job_state) + + if not hasattr(client, 'get_status'): + # Message queue implementation. + + # TODO: Very hacky now, refactor after Dannon merges in his + # message queue work, runners need the ability to disable + # check_watched_item like this and instead a callback needs to + # be issued post job recovery allowing a message queue + # consumer to be setup. + self.client_manager.ensure_has_job_completes_callback(self.__async_complete) + return job_state + status = client.get_status() except Exception: # An orphaned job was put into the queue at app startup, so remote server went down @@ -64,6 +77,29 @@ job_state.job_wrapper.change_state( model.Job.states.RUNNING ) return job_state + def __async_complete( self, final_status ): + job_id = final_status[ "job_id" ] + job_state = self.__find_watched_job( job_id ) + if not job_state: + # Probably finished too quickly, sleep and try again. + # Kind of a hack, why does monitor queue need to no wait + # get and sleep instead of doing a busy wait that would + # respond immediately. + sleep( 2 ) + job_state = self.__find_watched_job( job_id ) + if not job_state: + log.warn( "Failed to find job corresponding to final status %s in %s" % ( final_status, self.watched ) ) + else: + self.mark_as_finished( job_state ) + + def __find_watched_job( self, job_id ): + found_job = None + for async_job_state in self.watched: + if str( async_job_state.job_id ) == job_id: + found_job = async_job_state + break + return found_job + def queue_job(self, job_wrapper): job_destination = job_wrapper.job_destination @@ -184,15 +220,15 @@ return self.get_client( job_destination_params, job_id ) def get_client( self, job_destination_params, job_id ): - return self.client_manager.get_client( job_destination_params, job_id ) + return self.client_manager.get_client( job_destination_params, str( job_id ) ) def finish_job( self, job_state ): stderr = stdout = '' job_wrapper = job_state.job_wrapper try: client = self.get_client_from_state(job_state) + run_results = client.final_status() - run_results = client.raw_check_complete() stdout = run_results.get('stdout', '') stderr = run_results.get('stderr', '') exit_code = run_results.get('returncode', None) @@ -291,6 +327,10 @@ job_state.running = state == model.Job.states.RUNNING self.monitor_queue.put( job_state ) + def shutdown( self ): + super( LwrJobRunner, self ).shutdown() + self.client_manager.shutdown() + def __client_outputs( self, client, job_wrapper ): remote_work_dir_copy = LwrJobRunner.__remote_work_dir_copy( client ) if not remote_work_dir_copy: diff -r 7d649885e3259f3359e8c3816315c60e958c1518 -r 75b13d642f94b9ff6ec7c3c3713ad1177b243bd3 lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -8,6 +8,9 @@ from .setup_handler import build as build_setup_handler from .job_directory import RemoteJobDirectory +import logging +log = logging.getLogger(__name__) + CACHE_WAIT_SECONDS = 3 MAX_RETRY_COUNT = 5 RETRY_SLEEP_TIME = 0.1 @@ -272,6 +275,11 @@ launch_params["setup_params"] = dumps(setup_params) return self._raw_execute("launch", launch_params) + def final_status(self): + """ Return a dictionary summarizing final state of job. + """ + return self.raw_check_complete() + def kill(self): """ Cancel remote job, either removing from the queue or killing it. @@ -314,11 +322,11 @@ check_complete_response = self.raw_check_complete() # Older LWR instances won't set status so use 'complete', at some # point drop backward compatibility. - complete = self.check_complete(check_complete_response) - old_status = "complete" if complete else "running" - status = check_complete_response.get("status", old_status) + status = check_complete_response.get("status", None) # Bug in certains older LWR instances returned literal "status". - if status not in ["complete", "running", "queued"]: + if status in ["status", None]: + complete = self.check_complete(check_complete_response) + old_status = "complete" if complete else "running" status = old_status return status @@ -344,12 +352,12 @@ class MessageJobClient(BaseJobClient): - def __init__(self, destination_params, job_id, exchange): + def __init__(self, destination_params, job_id, client_manager): super(MessageJobClient, self).__init__(destination_params, job_id) if not self.job_directory: error_message = "Message-queue based LWR client requires destination define a remote job_directory to stage files into." raise Exception(error_message) - self.exchange = exchange + self.client_manager = client_manager def launch(self, command_line, requirements=[], remote_staging=[], job_config=None): """ @@ -368,7 +376,19 @@ # before queueing. setup_params = _setup_params_from_job_config(job_config) launch_params["setup_params"] = setup_params - return self.exchange.publish("setup", launch_params) + return self.client_manager.exchange.publish("setup", launch_params) + + def clean(self): + del self.client_manager.final_status_cache[self.job_id] + + def final_status(self): + final_status = self.client_manager.final_status_cache.get(self.job_id, None) + if final_status is None: + raise Exception("final_status() called before a final status was properly cached with cilent manager.") + return final_status + + def kill(self): + log.warn("Kill not yet implemented with message queue driven LWR jobs.") class InputCachingJobClient(JobClient): diff -r 7d649885e3259f3359e8c3816315c60e958c1518 -r 75b13d642f94b9ff6ec7c3c3713ad1177b243bd3 lib/galaxy/jobs/runners/lwr_client/manager.py --- a/lib/galaxy/jobs/runners/lwr_client/manager.py +++ b/lib/galaxy/jobs/runners/lwr_client/manager.py @@ -1,9 +1,9 @@ from abc import ABCMeta, abstractmethod +import threading try: from Queue import Queue except ImportError: from queue import Queue -from threading import Thread from os import getenv try: from urllib import urlencode @@ -87,25 +87,35 @@ self.url = kwds.get('url') self.manager_name = kwds.get("manager", "_default_") self.exchange = LwrExchange(self.url, self.manager_name) - self.thread = None + self.final_status_cache = {} + self.callback_lock = threading.Lock() + self.callback_thread = None self.active = True - def listen_for_job_completes(self, callback): - # TODO: Call this from LWR runner. - def callback_wrapper(body, message): - callback(body) - message.ack() + def ensure_has_job_completes_callback(self, callback): + with self.callback_lock: + if self.callback_thread is not None: + return - def run(): - self.exchange.consume("complete", callback_wrapper, check=self) + def callback_wrapper(body, message): + try: + if "job_id" in body: + self.final_status_cache[body["job_id"]] = body + callback(body) + except Exception: + log.exception("Failure processing job status update message.") + message.ack() - t = Thread( - name="lwr_client_%s_complete_callback" % self.manager_name, - target=run - ) - t.daemon = False # Lets not interrupt processing of this. - t.start() - self.thread = t + def run(): + self.exchange.consume("complete", callback_wrapper, check=self) + + thread = threading.Thread( + name="lwr_client_%s_complete_callback" % self.manager_name, + target=run + ) + thread.daemon = False # Lets not interrupt processing of this. + thread.start() + self.callback_thread = thread def shutdown(self): self.active = False @@ -115,7 +125,7 @@ def get_client(self, destination_params, job_id): destination_params = _parse_destination_params(destination_params) - return MessageJobClient(destination_params, job_id, self.exchange) + return MessageJobClient(destination_params, job_id, self) class ObjectStoreClientManager(object): @@ -259,7 +269,7 @@ self.num_transfer_threads = num_transfer_threads self.transfer_queue = Queue() for i in range(num_transfer_threads): - t = Thread(target=self._transfer_worker) + t = threading.Thread(target=self._transfer_worker) t.daemon = True t.start() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0