3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8388f17e522a/ Changeset: 8388f17e522a Branch: search User: Kyle Ellrott Date: 2014-01-02 20:37:37 Summary: Default merge Affected #: 467 files diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 .hgignore --- a/.hgignore +++ b/.hgignore @@ -60,6 +60,7 @@ job_conf.xml data_manager_conf.xml shed_data_manager_conf.xml +object_store_conf.xml config/* static/welcome.html.* static/welcome.html diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 config/plugins/visualizations/scatterplot/Gruntfile.js --- a/config/plugins/visualizations/scatterplot/Gruntfile.js +++ b/config/plugins/visualizations/scatterplot/Gruntfile.js @@ -9,7 +9,7 @@ // compile all hb templates into a single file in the build dir compile: { options: { - namespace: 'Templates', + namespace: 'scatterplot', processName : function( filepath ){ return filepath.match( /\w*\.handlebars/ )[0].replace( '.handlebars', '' ); } diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js --- a/config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js +++ b/config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js @@ -37,14 +37,19 @@ /** initialize requires a configuration Object containing a dataset Object */ initialize : function( attributes ){ //console.log( this + '.initialize, attributes:', attributes ); - if( !attributes || !attributes.config || !attributes.config.dataset ){ + if( !attributes || !attributes.config || !attributes.dataset ){ throw new Error( "ScatterplotView requires a configuration and dataset" ); } - this.dataset = attributes.config.dataset; + //console.log( 'config:', attributes.config ); + + this.dataset = attributes.dataset; //console.log( 'dataset:', this.dataset ); +//TODO: ScatterplotView -> ScatterplotDisplay, this.plotView -> this.display this.plotView = new ScatterplotView({ + dataset : attributes.dataset, config : attributes.config +//TODO: if data }); }, @@ -197,8 +202,8 @@ // parse the column values for both indeces (for the data fetch) and names (for the chart) var $dataControls = this.$el.find( '#data-control' ); var settings = { - xColumn : $dataControls.find( '[name="xColumn"]' ).val(), - yColumn : $dataControls.find( '[name="yColumn"]' ).val() + xColumn : Number( $dataControls.find( '[name="xColumn"]' ).val() ), + yColumn : Number( $dataControls.find( '[name="yColumn"]' ).val() ) }; if( $dataControls.find( '#include-id-checkbox' ).prop( 'checked' ) ){ settings.idColumn = $dataControls.find( '[name="idColumn"]' ).val(); @@ -229,9 +234,9 @@ }); ScatterplotConfigEditor.templates = { - mainLayout : Templates.editor, - dataControl : Templates.datacontrol, - chartControl : Templates.chartcontrol + mainLayout : scatterplot.editor, + dataControl : scatterplot.datacontrol, + chartControl : scatterplot.chartcontrol }; //============================================================================== diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 config/plugins/visualizations/scatterplot/src/scatterplot-display.js --- a/config/plugins/visualizations/scatterplot/src/scatterplot-display.js +++ b/config/plugins/visualizations/scatterplot/src/scatterplot-display.js @@ -10,14 +10,10 @@ //TODO: should be a view on visualization(revision) model defaults : { - dataset : { - }, metadata : { dataLines : undefined }, - ajaxFn : null, - pagination : { currPage : 0, perPage : 3000 @@ -48,6 +44,7 @@ initialize : function( attributes ){ this.config = _.extend( _.clone( this.defaults ), attributes.config || {}); + this.dataset = attributes.dataset; //console.debug( this + '.config:', this.config ); }, @@ -65,7 +62,7 @@ //console.debug( 'currPage', this.config.pagination.currPage ); var view = this; //TODO: very tied to datasets - should be generalized eventually - xhr = jQuery.getJSON( '/api/datasets/' + this.config.dataset.id, { + xhr = jQuery.getJSON( '/api/datasets/' + this.dataset.id, { data_type : 'raw_data', provider : 'dataset-column', limit : this.config.pagination.perPage, @@ -151,7 +148,7 @@ }, renderLineInfo : function( data ){ - var totalLines = this.config.dataset.metadata_data_lines || 'an unknown number of', + var totalLines = this.dataset.metadata_data_lines || 'an unknown number of', lineStart = ( this.config.pagination.currPage * this.config.pagination.perPage ), lineEnd = lineStart + data.length; return $( '<p/>' ).addClass( 'scatterplot-data-info' ) @@ -168,9 +165,9 @@ } //TODO: cache numPages/numLines in config var view = this, - dataLines = this.config.dataset.metadata_data_lines, + dataLines = this.dataset.metadata_data_lines, numPages = ( dataLines )?( Math.ceil( dataLines / this.config.pagination.perPage ) ):( undefined ); - //console.debug( 'data:', this.config.dataset.metadata_data_lines, 'numPages:', numPages ); + //console.debug( 'data:', this.dataset.metadata_data_lines, 'numPages:', numPages ); // prev next buttons var $prev = makePage$Li( 'Prev' ).click( function(){ @@ -207,9 +204,9 @@ } //TODO: cache numPages/numLines in config var view = this, - dataLines = this.config.dataset.metadata_data_lines, + dataLines = this.dataset.metadata_data_lines, numPages = ( dataLines )?( Math.ceil( dataLines / this.config.pagination.perPage ) ):( undefined ); - //console.debug( 'data:', this.config.dataset.metadata_data_lines, 'numPages:', numPages ); + //console.debug( 'data:', this.dataset.metadata_data_lines, 'numPages:', numPages ); // page numbers (as separate control) //var $paginationContainer = $( '<div/>' ).addClass( 'pagination-container' ), diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 config/plugins/visualizations/scatterplot/src/visualization-templates.html --- a/config/plugins/visualizations/scatterplot/src/visualization-templates.html +++ /dev/null @@ -1,197 +0,0 @@ -<script type="text/template" class="template-visualization" id="template-visualization-scatterplotControlForm"> -{{! main layout }} - -<h1>WHAAAAA?</h1> -<div class="scatterplot-container chart-container tabbable tabs-left"> - {{! tab buttons/headers using Bootstrap }} - <ul class="nav nav-tabs"> - {{! start with the data controls as the displayed tab }} - <li class="active"> - <a title="Use this tab to change which data are used" - href="#data-control" data-toggle="tab">Data Controls</a> - </li> - <li> - <a title="Use this tab to change how the chart is drawn" - href="#chart-control" data-toggle="tab" >Chart Controls</a> - </li> - <li> - <a title="This tab will display overall statistics for your data" - href="#stats-display" data-toggle="tab">Statistics</a> - </li> - <li> - <a title="This tab will display the chart" - href="#chart-display" data-toggle="tab">Chart</a> - - <div id="loading-indicator" style="display: none;"> - <img class="loading-img" src="{{loadingIndicatorImagePath}}" /> - <span class="loading-message">{{message}}</span> - </div> - </li> - </ul> - - {{! data form, chart config form, stats, and chart all get their own tab }} - <div class="tab-content"> - {{! ---------------------------- tab for data settings form }} - <div id="data-control" class="tab-pane active"> - {{! rendered separately }} - </div> - - {{! ---------------------------- tab for chart graphics control form }} - <div id="chart-control" class="tab-pane"> - {{! rendered separately }} - </div> - - {{! ---------------------------- tab for data statistics }} - <div id="stats-display" class="tab-pane"> - <p class="help-text">By column:</p> - <table id="chart-stats-table"> - <thead><th></th><th>X</th><th>Y</th></thead> - {{#each stats}} - <tr><td>{{name}}</td><td>{{xval}}</td><td>{{yval}}</td></tr> - </tr> - {{/each}} - </table> - </div> - - {{! ---------------------------- tab for actual chart }} - <div id="chart-display" class="tab-pane"> - <svg width="{{width}}" height="{{height}}"></svg> - </div> - - </div>{{! end .tab-content }} -</div>{{! end .chart-control }} -</script> - -<script type="text/template" class="template-visualization" id="template-visualization-dataControl"> - - <p class="help-text"> - Use the following controls to change the data used by the chart. - Use the 'Draw' button to render (or re-render) the chart with the current settings. - </p> - - {{! column selector containers }} - <div class="column-select"> - <label for="X-select">Data column for X: </label> - <select name="X" id="X-select"> - {{#each numericColumns}} - <option value="{{index}}">{{name}}</option> - {{/each}} - </select> - </div> - <div class="column-select"> - <label for="Y-select">Data column for Y: </label> - <select name="Y" id="Y-select"> - {{#each numericColumns}} - <option value="{{index}}">{{name}}</option> - {{/each}} - </select> - </div> - - {{! optional id column }} - <div id="include-id"> - <label for="include-id-checkbox">Include a third column as data point IDs?</label> - <input type="checkbox" name="include-id" id="include-id-checkbox" /> - <p class="help-text-small"> - These will be displayed (along with the x and y values) when you hover over - a data point. - </p> - </div> - <div class="column-select" style="display: none"> - <label for="ID-select">Data column for IDs: </label> - <select name="ID" id="ID-select"> - {{#each allColumns}} - <option value="{{index}}">{{name}}</option> - {{/each}} - </select> - </div> - - {{! if we're using generic column selection names ('column 1') - allow the user to use the first line }} - <div id="first-line-header" style="display: none;"> - <p>Possible headers: {{ possibleHeaders }} - </p> - <label for="first-line-header-checkbox">Use the above as column headers?</label> - <input type="checkbox" name="include-id" id="first-line-header-checkbox" - {{#if usePossibleHeaders }}checked="true"{{/if}}/> - <p class="help-text-small"> - It looks like Galaxy couldn't get proper column headers for this data. - Would you like to use the column headers above as column names to select columns? - </p> - </div> - - <input id="render-button" type="button" value="Draw" /> - <div class="clear"></div> -</script> - -<script type="text/template" class="template-visualization" id="template-visualization-chartControl"> - <p class="help-text"> - Use the following controls to how the chart is displayed. - The slide controls can be moved by the mouse or, if the 'handle' is in focus, your keyboard's arrow keys. - Move the focus between controls by using the tab or shift+tab keys on your keyboard. - Use the 'Draw' button to render (or re-render) the chart with the current settings. - </p> - - <div id="datapointSize" class="form-input numeric-slider-input"> - <label for="datapointSize">Size of data point: </label> - <div class="slider-output">{{datapointSize}}</div> - <div class="slider"></div> - <p class="form-help help-text-small"> - Size of the graphic representation of each data point - </p> - </div> - - <div id="animDuration" class="form-input checkbox-input"> - <label for="animate-chart">Animate chart transitions?: </label> - <input type="checkbox" id="animate-chart" - class="checkbox control"{{#if animDuration}} checked="true"{{/if}} /> - <p class="form-help help-text-small"> - Uncheck this to disable the animations used on the chart - </p> - </div> - - <div id="width" class="form-input numeric-slider-input"> - <label for="width">Chart width: </label> - <div class="slider-output">{{width}}</div> - <div class="slider"></div> - <p class="form-help help-text-small"> - (not including chart margins and axes) - </p> - </div> - - <div id="height" class="form-input numeric-slider-input"> - <label for="height">Chart height: </label> - <div class="slider-output">{{height}}</div> - <div class="slider"></div> - <p class="form-help help-text-small"> - (not including chart margins and axes) - </p> - </div> - - <div id="X-axis-label"class="text-input form-input"> - <label for="X-axis-label">Re-label the X axis: </label> - <input type="text" name="X-axis-label" id="X-axis-label" value="{{xLabel}}" /> - <p class="form-help help-text-small"></p> - </div> - - <div id="Y-axis-label" class="text-input form-input"> - <label for="Y-axis-label">Re-label the Y axis: </label> - <input type="text" name="Y-axis-label" id="Y-axis-label" value="{{yLabel}}" /> - <p class="form-help help-text-small"></p> - </div> - - <input id="render-button" type="button" value="Draw" /> -</script> - -<script type="text/template" class="template-visualization" id="template-visualization-statsDisplay"> - <p class="help-text">By column:</p> - <table id="chart-stats-table"> - <thead><th></th><th>X</th><th>Y</th></thead> - {{#each stats}} - <tr><td>{{name}}</td><td>{{xval}}</td><td>{{yval}}</td></tr> - </tr> - {{/each}} - </table> -</script> - -<script type="text/template" class="template-visualization" id="template-visualization-chartDisplay"> - <svg width="{{width}}" height="{{height}}"></svg> -</script> diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 config/plugins/visualizations/scatterplot/static/scatterplot-edit.js --- a/config/plugins/visualizations/scatterplot/static/scatterplot-edit.js +++ b/config/plugins/visualizations/scatterplot/static/scatterplot-edit.js @@ -1,1 +1,1 @@ -function scatterplot(a,b,c){function d(){var a={v:{},h:{}};return a.v.lines=p.selectAll("line.v-grid-line").data(m.x.ticks(q.x.fn.ticks()[0])),a.v.lines.enter().append("svg:line").classed("grid-line v-grid-line",!0),a.v.lines.attr("x1",m.x).attr("x2",m.x).attr("y1",0).attr("y2",b.height),a.v.lines.exit().remove(),a.h.lines=p.selectAll("line.h-grid-line").data(m.y.ticks(q.y.fn.ticks()[0])),a.h.lines.enter().append("svg:line").classed("grid-line h-grid-line",!0),a.h.lines.attr("x1",0).attr("x2",b.width).attr("y1",m.y).attr("y2",m.y),a.h.lines.exit().remove(),a}function e(){return t.attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",function(a,b){return m.y(k(a,b))}).style("display","block").filter(function(){var a=d3.select(this).attr("cx"),c=d3.select(this).attr("cy");return 0>a||a>b.width?!0:0>c||c>b.height?!0:!1}).style("display","none")}function f(){q.redraw(),e(),s=d(),$(".chart-info-box").remove(),$(o.node()).trigger("zoom.scatterplot",[])}function g(a,c,d){return c+=8,$(['<div class="chart-info-box" style="position: absolute">',b.idColumn?"<div>"+d[b.idColumn]+"</div>":"","<div>",j(d),"</div>","<div>",k(d),"</div>","</div>"].join("")).css({top:a,left:c,"z-index":2})}var h=function(a,b){return"translate("+a+","+b+")"},i=function(a,b,c){return"rotate("+a+","+b+","+c+")"},j=function(a){return a[b.xColumn]},k=function(a){return a[b.yColumn]},l={x:{extent:d3.extent(c,j)},y:{extent:d3.extent(c,k)}},m={x:d3.scale.linear().domain(l.x.extent).range([0,b.width]),y:d3.scale.linear().domain(l.y.extent).range([b.height,0])},n=d3.behavior.zoom().x(m.x).y(m.y).scaleExtent([1,10]),o=d3.select(a).attr("class","scatterplot").attr("width","100%").attr("height",b.height+(b.margin.top+b.margin.bottom)),p=o.append("g").attr("class","content").attr("transform",h(b.margin.left,b.margin.top)).call(n);p.append("rect").attr("class","zoom-rect").attr("width",b.width).attr("height",b.height).style("fill","transparent");var q={x:{},y:{}};q.x.fn=d3.svg.axis().orient("bottom").scale(m.x).ticks(b.x.ticks).tickFormat(d3.format("s")),q.y.fn=d3.svg.axis().orient("left").scale(m.y).ticks(b.y.ticks).tickFormat(d3.format("s")),q.x.g=p.append("g").attr("class","x axis").attr("transform",h(0,b.height)).call(q.x.fn),q.y.g=p.append("g").attr("class","y axis").call(q.y.fn);var r=4;q.x.label=o.append("text").attr("class","axis-label").text(b.x.label).attr("text-anchor","middle").attr("dominant-baseline","text-after-edge").attr("x",b.width/2+b.margin.left).attr("y",b.height+b.margin.bottom+b.margin.top-r),q.y.label=o.append("text").attr("class","axis-label").text(b.y.label).attr("text-anchor","middle").attr("dominant-baseline","text-before-edge").attr("x",r).attr("y",b.height/2).attr("transform",i(-90,r,b.height/2)),q.redraw=function(){o.select(".x.axis").call(q.x.fn),o.select(".y.axis").call(q.y.fn)};var s=d(),t=p.selectAll(".glyph").data(c).enter().append("svg:circle").classed("glyph",!0).attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",b.height).attr("r",0);t.transition().duration(b.animDuration).attr("cy",function(a,b){return m.y(k(a,b))}).attr("r",b.datapointSize),n.on("zoom",f),t.on("mouseover",function(a,c){var d=d3.select(this);d.style("fill","red").style("fill-opacity",1),p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")-b.datapointSize).attr("y1",d.attr("cy")).attr("x2",0).attr("y2",d.attr("cy")).classed("hoverline",!0),d.attr("cy")<b.height&&p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")).attr("y1",+d.attr("cy")+b.datapointSize).attr("x2",d.attr("cx")).attr("y2",b.height).classed("hoverline",!0);var e=this.getBoundingClientRect();$("body").append(g(e.top,e.right,a)),$(o.node()).trigger("mouseover-datapoint.scatterplot",[this,a,c])}),t.on("mouseout",function(){d3.select(this).style("fill","black").style("fill-opacity",.2),p.selectAll(".hoverline").remove(),$(".chart-info-box").remove()})}this.Templates=this.Templates||{},this.Templates.chartcontrol=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f,g="",h="function",i=this.escapeExpression;return g+='<p class="help-text">\n Use the following controls to how the chart is displayed.\n The slide controls can be moved by the mouse or, if the \'handle\' is in focus, your keyboard\'s arrow keys.\n Move the focus between controls by using the tab or shift+tab keys on your keyboard.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n</p>\n\n<div data-config-key="datapointSize" class="form-input numeric-slider-input">\n <label for="datapointSize">Size of data point: </label>\n <div class="slider-output">',(f=c.datapointSize)?f=f.call(b,{hash:{},data:e}):(f=b.datapointSize,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n Size of the graphic representation of each data point\n </p>\n</div>\n\n<div data-config-key="width" class="form-input numeric-slider-input">\n <label for="width">Chart width: </label>\n <div class="slider-output">',(f=c.width)?f=f.call(b,{hash:{},data:e}):(f=b.width,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n (not including chart margins and axes)\n </p>\n</div>\n\n<div data-config-key="height" class="form-input numeric-slider-input">\n <label for="height">Chart height: </label>\n <div class="slider-output">',(f=c.height)?f=f.call(b,{hash:{},data:e}):(f=b.height,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n (not including chart margins and axes)\n </p>\n</div>\n\n<div data-config-key="X-axis-label"class="text-input form-input">\n <label for="X-axis-label">Re-label the X axis: </label>\n <input type="text" name="X-axis-label" id="X-axis-label" value="'+i((f=b.x,f=null==f||f===!1?f:f.label,typeof f===h?f.apply(b):f))+'" />\n <p class="form-help help-text-small"></p>\n</div>\n\n<div data-config-key="Y-axis-label" class="text-input form-input">\n <label for="Y-axis-label">Re-label the Y axis: </label>\n <input type="text" name="Y-axis-label" id="Y-axis-label" value="'+i((f=b.y,f=null==f||f===!1?f:f.label,typeof f===h?f.apply(b):f))+'" />\n <p class="form-help help-text-small"></p>\n</div>\n\n<button class="render-button btn btn-primary active">Draw</button>\n'}),this.Templates.datacontrol=Handlebars.template(function(a,b,c,d,e){function f(a,b){var d,e="";return e+='\n <option value="',(d=c.index)?d=d.call(a,{hash:{},data:b}):(d=a.index,d=typeof d===j?d.apply(a):d),e+=k(d)+'">',(d=c.name)?d=d.call(a,{hash:{},data:b}):(d=a.name,d=typeof d===j?d.apply(a):d),e+=k(d)+"</option>\n "}function g(){return'checked="true"'}this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var h,i="",j="function",k=this.escapeExpression,l=this;return i+='<p class="help-text">\n Use the following controls to change the data used by the chart.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n</p>\n\n\n<div class="column-select">\n <label>Data column for X: </label>\n <select name="xColumn">\n ',h=c.each.call(b,b.numericColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n<div class="column-select">\n <label>Data column for Y: </label>\n <select name="yColumn">\n ',h=c.each.call(b,b.numericColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n\n\n<div id="include-id">\n <label for="include-id-checkbox">Include a third column as data point IDs?</label>\n <input type="checkbox" name="include-id" id="include-id-checkbox" />\n <p class="help-text-small">\n These will be displayed (along with the x and y values) when you hover over\n a data point.\n </p>\n</div>\n<div class="column-select" style="display: none">\n <label for="ID-select">Data column for IDs: </label>\n <select name="idColumn">\n ',h=c.each.call(b,b.allColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n\n\n<div id="first-line-header" style="display: none;">\n <p>Possible headers: ',(h=c.possibleHeaders)?h=h.call(b,{hash:{},data:e}):(h=b.possibleHeaders,h=typeof h===j?h.apply(b):h),i+=k(h)+'\n </p>\n <label for="first-line-header-checkbox">Use the above as column headers?</label>\n <input type="checkbox" name="include-id" id="first-line-header-checkbox"\n ',h=c["if"].call(b,b.usePossibleHeaders,{hash:{},inverse:l.noop,fn:l.program(3,g,e),data:e}),(h||0===h)&&(i+=h),i+='/>\n <p class="help-text-small">\n It looks like Galaxy couldn\'t get proper column headers for this data.\n Would you like to use the column headers above as column names to select columns?\n </p>\n</div>\n\n<button class="render-button btn btn-primary active">Draw</button>\n'}),this.Templates.editor=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f="";return f+='<div class="scatterplot-editor tabbable tabs-left">\n \n <ul class="nav nav-tabs">\n \n <li class="active">\n <a title="Use this tab to change which data are used"\n href="#data-control" data-toggle="tab">Data Controls</a>\n </li>\n <li>\n <a title="Use this tab to change how the chart is drawn"\n href="#chart-control" data-toggle="tab" >Chart Controls</a>\n </li>\n \n <li class="disabled">\n <a title="This tab will display the chart"\n href="#chart-display" data-toggle="tab">Chart</a>\n </li>\n </ul>\n\n \n <div class="tab-content">\n \n <div id="data-control" class="scatterplot-config-control tab-pane active">\n \n </div>\n \n \n <div id="chart-control" class="scatterplot-config-control tab-pane">\n \n </div>\n\n \n <div id="chart-display" class="scatterplot-display tab-pane"></div>\n\n </div>\n</div>\n'});var ScatterplotConfigEditor=BaseView.extend(LoggableMixin).extend({className:"scatterplot-control-form",initialize:function(a){if(!a||!a.config||!a.config.dataset)throw new Error("ScatterplotView requires a configuration and dataset");this.dataset=a.config.dataset,this.plotView=new ScatterplotView({config:a.config})},render:function(){return this.$el.append(ScatterplotConfigEditor.templates.mainLayout({})),this.$el.find("#data-control").append(this._render_dataControl()),this._render_chartControls(this.$el.find("#chart-control")),this._render_chartDisplay(),this.$el.find("[title]").tooltip(),this},_render_dataControl:function(){var a=this.dataset,b=_.map(a.metadata_column_types,function(b,c){var d={index:c,type:b,name:"column "+(c+1)};return a.metadata_column_names&&a.metadata_column_names[c]&&(d.name=a.metadata_column_names[c]),d}),c=_.filter(b,function(a){return"int"===a.type||"float"===a.type});2>c&&(c=b);var d=this.$el.find(".tab-pane#data-control");return d.html(ScatterplotConfigEditor.templates.dataControl({allColumns:b,numericColumns:c})),d.find('[name="xColumn"]').val(this.plotView.config.xColumn||c[0].index),d.find('[name="yColumn"]').val(this.plotView.config.yColumn||c[1].index),void 0!==this.plotView.config.idColumn&&(d.find("#include-id-checkbox").prop("checked",!0).trigger("change"),d.find('select[name="idColumn"]').val(this.plotView.config.idColumn)),d},_render_chartControls:function(a){function b(){var a=$(this);a.siblings(".slider-output").text(a.slider("value"))}a.html(ScatterplotConfigEditor.templates.chartControl(this.plotView.config));var c=this,d={datapointSize:{min:2,max:10,step:1},width:{min:200,max:800,step:20},height:{min:200,max:800,step:20}};return a.find(".numeric-slider-input").each(function(){var a=$(this),e=a.attr("data-config-key"),f=_.extend(d[e],{value:c.plotView.config[e],change:b,slide:b});a.find(".slider").slider(f)}),this.dataset.metadata_column_names,a},_render_chartDisplay:function(){var a=this.$el.find(".tab-pane#chart-display");return this.plotView.setElement(a),this.plotView.render(),a},events:{"change #include-id-checkbox":"toggleThirdColumnSelector","click #data-control .render-button":"renderChart","click #chart-control .render-button":"renderChart"},toggleThirdColumnSelector:function(){this.$el.find('select[name="idColumn"]').parent().toggle()},renderChart:function(){this.$el.find(".nav li.disabled").removeClass("disabled"),this.updateConfigWithDataSettings(),this.updateConfigWithChartSettings(),this.$el.find("ul.nav").find('a[href="#chart-display"]').tab("show"),this.plotView.fetchData()},updateConfigWithDataSettings:function(){var a=this.$el.find("#data-control"),b={xColumn:a.find('[name="xColumn"]').val(),yColumn:a.find('[name="yColumn"]').val()};return a.find("#include-id-checkbox").prop("checked")&&(b.idColumn=a.find('[name="idColumn"]').val()),_.extend(this.plotView.config,b)},updateConfigWithChartSettings:function(){var a=this.plotView,b=this.$el.find("#chart-control");return["datapointSize","width","height"].forEach(function(c){a.config[c]=b.find('.numeric-slider-input[data-config-key="'+c+'"]').find(".slider").slider("value")}),a.config.x.label=b.find('input[name="X-axis-label"]').val(),a.config.y.label=b.find('input[name="Y-axis-label"]').val(),a.config},toString:function(){return"ScatterplotConfigEditor("+(this.dataset?this.dataset.id:"")+")"}});ScatterplotConfigEditor.templates={mainLayout:Templates.editor,dataControl:Templates.datacontrol,chartControl:Templates.chartcontrol};var ScatterplotView=Backbone.View.extend({defaults:{dataset:{},metadata:{dataLines:void 0},ajaxFn:null,pagination:{currPage:0,perPage:3e3},width:400,height:400,margin:{top:16,right:16,bottom:40,left:54},x:{ticks:10,label:"X"},y:{ticks:10,label:"Y"},datapointSize:4,animDuration:500},initialize:function(a){this.config=_.extend(_.clone(this.defaults),a.config||{})},updateConfig:function(a){this.config=this.config||{},_.extend(this.config,a)},fetchData:function(){this.showLoadingIndicator("getting data");var a=this;return xhr=jQuery.getJSON("/api/datasets/"+this.config.dataset.id,{data_type:"raw_data",provider:"dataset-column",limit:this.config.pagination.perPage,offset:this.config.pagination.currPage*this.config.pagination.perPage}),xhr.done(function(b){a.renderData(b.data)}),xhr.fail(function(a,b,c){alert("Error loading data:\n"+a.responseText),console.error(a,b,c)}),xhr.always(function(){a.hideLoadingIndicator()}),xhr},render:function(a){return this.$el.addClass("scatterplot-display").html(['<div class="controls clear"></div>','<div class="loading-indicator">','<span class="fa fa-spinner fa-spin"></span>','<span class="loading-indicator-message"></span>',"</div>","<svg/>",'<div class="stats-display"></div>'].join("")),this.$el.children().hide(),a&&this.renderData(a),this},showLoadingIndicator:function(a,b){a=a||"",b=b||"fast";var c=this.$el.find(".loading-indicator");a&&c.find(".loading-indicator-message").text(a),c.is(":visible")||(this.toggleStats(!1),c.css({left:this.config.width/2,top:this.config.height/2}).show())},hideLoadingIndicator:function(a){a=a||"fast",this.$el.find(".loading-indicator").hide()},renderData:function(a){this.$el.find(".controls").empty().append(this.renderControls(a)).show(),this.renderPlot(a),this.getStats(a)},renderControls:function(a){var b=this,c=$('<div class="left"></div>'),d=$('<div class="right"></div>');return c.append([this.renderPrevNext(a),this.renderPagination(a)]),d.append([this.renderLineInfo(a),$("<button>Stats</button>").addClass("stats-toggle-btn").click(function(){b.toggleStats()}),$("<button>Redraw</button>").addClass("rerender-btn").click(function(){b.renderPlot(a)})]),[c,d]},renderLineInfo:function(a){var b=this.config.dataset.metadata_data_lines||"an unknown number of",c=this.config.pagination.currPage*this.config.pagination.perPage,d=c+a.length;return $("<p/>").addClass("scatterplot-data-info").text(["Displaying lines",c+1,"to",d,"of",b,"lines"].join(" "))},renderPrevNext:function(a){function b(a){return $(['<li><a href="javascript:void(0);">',a,"</a></li>"].join(""))}if(!a||0===this.config.pagination.currPage&&a.length<this.config.pagination.perPage)return null;var c=this,d=this.config.dataset.metadata_data_lines,e=d?Math.ceil(d/this.config.pagination.perPage):void 0,f=b("Prev").click(function(){c.config.pagination.currPage>0&&(c.config.pagination.currPage-=1,c.fetchData())}),g=b("Next").click(function(){(!e||c.config.pagination.currPage<e-1)&&(c.config.pagination.currPage+=1,c.fetchData())}),h=$("<ul/>").addClass("pagination data-prev-next").append([f,g]);return 0===c.config.pagination.currPage&&f.addClass("disabled"),e&&c.config.pagination.currPage===e-1&&g.addClass("disabled"),h},renderPagination:function(a){function b(a){return $(['<li><a href="javascript:void(0);">',a,"</a></li>"].join(""))}function c(){d.config.pagination.currPage=$(this).data("page"),d.fetchData()}if(!a||0===this.config.pagination.currPage&&a.length<this.config.pagination.perPage)return null;for(var d=this,e=this.config.dataset.metadata_data_lines,f=e?Math.ceil(e/this.config.pagination.perPage):void 0,g=$("<ul/>").addClass("pagination data-pages"),h=0;f>h;h+=1){var i=b(h+1).attr("data-page",h).click(c);h===this.config.pagination.currPage&&i.addClass("active"),g.append(i)}return g},renderPlot:function(a){this.toggleStats(!1);var b=this.$el.find("svg");b.off().empty().show(),scatterplot(b.get(0),this.config,a)},getStats:function(a){var b=this;meanWorker=new Worker("/plugins/visualizations/scatterplot/static/worker-stats.js"),meanWorker.postMessage({data:a,keys:[this.config.xColumn,this.config.yColumn]}),meanWorker.onerror=function(){meanWorker.terminate()},meanWorker.onmessage=function(a){b.renderStats(a.data)}},renderStats:function(a){var b=this.$el.find(".stats-display"),c=this.config.x.label,d=this.config.y.label,e=$("<table/>").addClass("table").append(["<thead><th></th><th>",c,"</th><th>",d,"</th></thead>"].join("")).append(_.map(a,function(a,b){return $(["<tr><td>",b,"</td><td>",a[0],"</td><td>",a[1],"</td></tr>"].join(""))}));b.empty().append(e)},toggleStats:function(a){var b=this.$el.find(".stats-display");a=void 0===a?b.is(":hidden"):a,a?(this.$el.find("svg").hide(),b.show(),this.$el.find(".controls .stats-toggle-btn").text("Plot")):(b.hide(),this.$el.find("svg").show(),this.$el.find(".controls .stats-toggle-btn").text("Stats"))},toString:function(){return"ScatterplotView()"}}); \ No newline at end of file +function scatterplot(a,b,c){function d(){var a={v:{},h:{}};return a.v.lines=p.selectAll("line.v-grid-line").data(m.x.ticks(q.x.fn.ticks()[0])),a.v.lines.enter().append("svg:line").classed("grid-line v-grid-line",!0),a.v.lines.attr("x1",m.x).attr("x2",m.x).attr("y1",0).attr("y2",b.height),a.v.lines.exit().remove(),a.h.lines=p.selectAll("line.h-grid-line").data(m.y.ticks(q.y.fn.ticks()[0])),a.h.lines.enter().append("svg:line").classed("grid-line h-grid-line",!0),a.h.lines.attr("x1",0).attr("x2",b.width).attr("y1",m.y).attr("y2",m.y),a.h.lines.exit().remove(),a}function e(){return t.attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",function(a,b){return m.y(k(a,b))}).style("display","block").filter(function(){var a=d3.select(this).attr("cx"),c=d3.select(this).attr("cy");return 0>a||a>b.width?!0:0>c||c>b.height?!0:!1}).style("display","none")}function f(){q.redraw(),e(),s=d(),$(".chart-info-box").remove(),$(o.node()).trigger("zoom.scatterplot",[])}function g(a,c,d){return c+=8,$(['<div class="chart-info-box" style="position: absolute">',b.idColumn?"<div>"+d[b.idColumn]+"</div>":"","<div>",j(d),"</div>","<div>",k(d),"</div>","</div>"].join("")).css({top:a,left:c,"z-index":2})}var h=function(a,b){return"translate("+a+","+b+")"},i=function(a,b,c){return"rotate("+a+","+b+","+c+")"},j=function(a){return a[b.xColumn]},k=function(a){return a[b.yColumn]},l={x:{extent:d3.extent(c,j)},y:{extent:d3.extent(c,k)}},m={x:d3.scale.linear().domain(l.x.extent).range([0,b.width]),y:d3.scale.linear().domain(l.y.extent).range([b.height,0])},n=d3.behavior.zoom().x(m.x).y(m.y).scaleExtent([1,10]),o=d3.select(a).attr("class","scatterplot").attr("width","100%").attr("height",b.height+(b.margin.top+b.margin.bottom)),p=o.append("g").attr("class","content").attr("transform",h(b.margin.left,b.margin.top)).call(n);p.append("rect").attr("class","zoom-rect").attr("width",b.width).attr("height",b.height).style("fill","transparent");var q={x:{},y:{}};q.x.fn=d3.svg.axis().orient("bottom").scale(m.x).ticks(b.x.ticks).tickFormat(d3.format("s")),q.y.fn=d3.svg.axis().orient("left").scale(m.y).ticks(b.y.ticks).tickFormat(d3.format("s")),q.x.g=p.append("g").attr("class","x axis").attr("transform",h(0,b.height)).call(q.x.fn),q.y.g=p.append("g").attr("class","y axis").call(q.y.fn);var r=4;q.x.label=o.append("text").attr("class","axis-label").text(b.x.label).attr("text-anchor","middle").attr("dominant-baseline","text-after-edge").attr("x",b.width/2+b.margin.left).attr("y",b.height+b.margin.bottom+b.margin.top-r),q.y.label=o.append("text").attr("class","axis-label").text(b.y.label).attr("text-anchor","middle").attr("dominant-baseline","text-before-edge").attr("x",r).attr("y",b.height/2).attr("transform",i(-90,r,b.height/2)),q.redraw=function(){o.select(".x.axis").call(q.x.fn),o.select(".y.axis").call(q.y.fn)};var s=d(),t=p.selectAll(".glyph").data(c).enter().append("svg:circle").classed("glyph",!0).attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",b.height).attr("r",0);t.transition().duration(b.animDuration).attr("cy",function(a,b){return m.y(k(a,b))}).attr("r",b.datapointSize),n.on("zoom",f),t.on("mouseover",function(a,c){var d=d3.select(this);d.style("fill","red").style("fill-opacity",1),p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")-b.datapointSize).attr("y1",d.attr("cy")).attr("x2",0).attr("y2",d.attr("cy")).classed("hoverline",!0),d.attr("cy")<b.height&&p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")).attr("y1",+d.attr("cy")+b.datapointSize).attr("x2",d.attr("cx")).attr("y2",b.height).classed("hoverline",!0);var e=this.getBoundingClientRect();$("body").append(g(e.top,e.right,a)),$(o.node()).trigger("mouseover-datapoint.scatterplot",[this,a,c])}),t.on("mouseout",function(){d3.select(this).style("fill","black").style("fill-opacity",.2),p.selectAll(".hoverline").remove(),$(".chart-info-box").remove()})}this.Templates=this.Templates||{},this.Templates.chartcontrol=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f,g="",h="function",i=this.escapeExpression;return g+='<p class="help-text">\n Use the following controls to how the chart is displayed.\n The slide controls can be moved by the mouse or, if the \'handle\' is in focus, your keyboard\'s arrow keys.\n Move the focus between controls by using the tab or shift+tab keys on your keyboard.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n</p>\n\n<div data-config-key="datapointSize" class="form-input numeric-slider-input">\n <label for="datapointSize">Size of data point: </label>\n <div class="slider-output">',(f=c.datapointSize)?f=f.call(b,{hash:{},data:e}):(f=b.datapointSize,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n Size of the graphic representation of each data point\n </p>\n</div>\n\n<div data-config-key="width" class="form-input numeric-slider-input">\n <label for="width">Chart width: </label>\n <div class="slider-output">',(f=c.width)?f=f.call(b,{hash:{},data:e}):(f=b.width,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n (not including chart margins and axes)\n </p>\n</div>\n\n<div data-config-key="height" class="form-input numeric-slider-input">\n <label for="height">Chart height: </label>\n <div class="slider-output">',(f=c.height)?f=f.call(b,{hash:{},data:e}):(f=b.height,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n (not including chart margins and axes)\n </p>\n</div>\n\n<div data-config-key="X-axis-label"class="text-input form-input">\n <label for="X-axis-label">Re-label the X axis: </label>\n <input type="text" name="X-axis-label" id="X-axis-label" value="'+i((f=b.x,f=null==f||f===!1?f:f.label,typeof f===h?f.apply(b):f))+'" />\n <p class="form-help help-text-small"></p>\n</div>\n\n<div data-config-key="Y-axis-label" class="text-input form-input">\n <label for="Y-axis-label">Re-label the Y axis: </label>\n <input type="text" name="Y-axis-label" id="Y-axis-label" value="'+i((f=b.y,f=null==f||f===!1?f:f.label,typeof f===h?f.apply(b):f))+'" />\n <p class="form-help help-text-small"></p>\n</div>\n\n<button class="render-button btn btn-primary active">Draw</button>\n'}),this.Templates.datacontrol=Handlebars.template(function(a,b,c,d,e){function f(a,b){var d,e="";return e+='\n <option value="',(d=c.index)?d=d.call(a,{hash:{},data:b}):(d=a.index,d=typeof d===j?d.apply(a):d),e+=k(d)+'">',(d=c.name)?d=d.call(a,{hash:{},data:b}):(d=a.name,d=typeof d===j?d.apply(a):d),e+=k(d)+"</option>\n "}function g(){return'checked="true"'}this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var h,i="",j="function",k=this.escapeExpression,l=this;return i+='<p class="help-text">\n Use the following controls to change the data used by the chart.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n</p>\n\n\n<div class="column-select">\n <label>Data column for X: </label>\n <select name="xColumn">\n ',h=c.each.call(b,b.numericColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n<div class="column-select">\n <label>Data column for Y: </label>\n <select name="yColumn">\n ',h=c.each.call(b,b.numericColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n\n\n<div id="include-id">\n <label for="include-id-checkbox">Include a third column as data point IDs?</label>\n <input type="checkbox" name="include-id" id="include-id-checkbox" />\n <p class="help-text-small">\n These will be displayed (along with the x and y values) when you hover over\n a data point.\n </p>\n</div>\n<div class="column-select" style="display: none">\n <label for="ID-select">Data column for IDs: </label>\n <select name="idColumn">\n ',h=c.each.call(b,b.allColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n\n\n<div id="first-line-header" style="display: none;">\n <p>Possible headers: ',(h=c.possibleHeaders)?h=h.call(b,{hash:{},data:e}):(h=b.possibleHeaders,h=typeof h===j?h.apply(b):h),i+=k(h)+'\n </p>\n <label for="first-line-header-checkbox">Use the above as column headers?</label>\n <input type="checkbox" name="include-id" id="first-line-header-checkbox"\n ',h=c["if"].call(b,b.usePossibleHeaders,{hash:{},inverse:l.noop,fn:l.program(3,g,e),data:e}),(h||0===h)&&(i+=h),i+='/>\n <p class="help-text-small">\n It looks like Galaxy couldn\'t get proper column headers for this data.\n Would you like to use the column headers above as column names to select columns?\n </p>\n</div>\n\n<button class="render-button btn btn-primary active">Draw</button>\n'}),this.Templates.editor=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f="";return f+='<div class="scatterplot-editor tabbable tabs-left">\n \n <ul class="nav nav-tabs">\n \n <li class="active">\n <a title="Use this tab to change which data are used"\n href="#data-control" data-toggle="tab">Data Controls</a>\n </li>\n <li>\n <a title="Use this tab to change how the chart is drawn"\n href="#chart-control" data-toggle="tab" >Chart Controls</a>\n </li>\n \n <li class="disabled">\n <a title="This tab will display the chart"\n href="#chart-display" data-toggle="tab">Chart</a>\n </li>\n </ul>\n\n \n <div class="tab-content">\n \n <div id="data-control" class="scatterplot-config-control tab-pane active">\n \n </div>\n \n \n <div id="chart-control" class="scatterplot-config-control tab-pane">\n \n </div>\n\n \n <div id="chart-display" class="scatterplot-display tab-pane"></div>\n\n </div>\n</div>\n'});var ScatterplotConfigEditor=BaseView.extend(LoggableMixin).extend({className:"scatterplot-control-form",initialize:function(a){if(!a||!a.config||!a.dataset)throw new Error("ScatterplotView requires a configuration and dataset");this.dataset=a.dataset,this.plotView=new ScatterplotView({dataset:a.dataset,config:a.config})},render:function(){return this.$el.append(ScatterplotConfigEditor.templates.mainLayout({})),this.$el.find("#data-control").append(this._render_dataControl()),this._render_chartControls(this.$el.find("#chart-control")),this._render_chartDisplay(),this.$el.find("[title]").tooltip(),this},_render_dataControl:function(){var a=this.dataset,b=_.map(a.metadata_column_types,function(b,c){var d={index:c,type:b,name:"column "+(c+1)};return a.metadata_column_names&&a.metadata_column_names[c]&&(d.name=a.metadata_column_names[c]),d}),c=_.filter(b,function(a){return"int"===a.type||"float"===a.type});2>c&&(c=b);var d=this.$el.find(".tab-pane#data-control");return d.html(ScatterplotConfigEditor.templates.dataControl({allColumns:b,numericColumns:c})),d.find('[name="xColumn"]').val(this.plotView.config.xColumn||c[0].index),d.find('[name="yColumn"]').val(this.plotView.config.yColumn||c[1].index),void 0!==this.plotView.config.idColumn&&(d.find("#include-id-checkbox").prop("checked",!0).trigger("change"),d.find('select[name="idColumn"]').val(this.plotView.config.idColumn)),d},_render_chartControls:function(a){function b(){var a=$(this);a.siblings(".slider-output").text(a.slider("value"))}a.html(ScatterplotConfigEditor.templates.chartControl(this.plotView.config));var c=this,d={datapointSize:{min:2,max:10,step:1},width:{min:200,max:800,step:20},height:{min:200,max:800,step:20}};return a.find(".numeric-slider-input").each(function(){var a=$(this),e=a.attr("data-config-key"),f=_.extend(d[e],{value:c.plotView.config[e],change:b,slide:b});a.find(".slider").slider(f)}),this.dataset.metadata_column_names,a},_render_chartDisplay:function(){var a=this.$el.find(".tab-pane#chart-display");return this.plotView.setElement(a),this.plotView.render(),a},events:{"change #include-id-checkbox":"toggleThirdColumnSelector","click #data-control .render-button":"renderChart","click #chart-control .render-button":"renderChart"},toggleThirdColumnSelector:function(){this.$el.find('select[name="idColumn"]').parent().toggle()},renderChart:function(){this.$el.find(".nav li.disabled").removeClass("disabled"),this.updateConfigWithDataSettings(),this.updateConfigWithChartSettings(),this.$el.find("ul.nav").find('a[href="#chart-display"]').tab("show"),this.plotView.fetchData()},updateConfigWithDataSettings:function(){var a=this.$el.find("#data-control"),b={xColumn:Number(a.find('[name="xColumn"]').val()),yColumn:Number(a.find('[name="yColumn"]').val())};return a.find("#include-id-checkbox").prop("checked")&&(b.idColumn=a.find('[name="idColumn"]').val()),_.extend(this.plotView.config,b)},updateConfigWithChartSettings:function(){var a=this.plotView,b=this.$el.find("#chart-control");return["datapointSize","width","height"].forEach(function(c){a.config[c]=b.find('.numeric-slider-input[data-config-key="'+c+'"]').find(".slider").slider("value")}),a.config.x.label=b.find('input[name="X-axis-label"]').val(),a.config.y.label=b.find('input[name="Y-axis-label"]').val(),a.config},toString:function(){return"ScatterplotConfigEditor("+(this.dataset?this.dataset.id:"")+")"}});ScatterplotConfigEditor.templates={mainLayout:Templates.editor,dataControl:Templates.datacontrol,chartControl:Templates.chartcontrol};var ScatterplotView=Backbone.View.extend({defaults:{metadata:{dataLines:void 0},pagination:{currPage:0,perPage:3e3},width:400,height:400,margin:{top:16,right:16,bottom:40,left:54},x:{ticks:10,label:"X"},y:{ticks:10,label:"Y"},datapointSize:4,animDuration:500},initialize:function(a){this.config=_.extend(_.clone(this.defaults),a.config||{}),this.dataset=a.dataset},updateConfig:function(a){this.config=this.config||{},_.extend(this.config,a)},fetchData:function(){this.showLoadingIndicator("getting data");var a=this;return xhr=jQuery.getJSON("/api/datasets/"+this.dataset.id,{data_type:"raw_data",provider:"dataset-column",limit:this.config.pagination.perPage,offset:this.config.pagination.currPage*this.config.pagination.perPage}),xhr.done(function(b){a.renderData(b.data)}),xhr.fail(function(a,b,c){alert("Error loading data:\n"+a.responseText),console.error(a,b,c)}),xhr.always(function(){a.hideLoadingIndicator()}),xhr},render:function(a){return this.$el.addClass("scatterplot-display").html(['<div class="controls clear"></div>','<div class="loading-indicator">','<span class="fa fa-spinner fa-spin"></span>','<span class="loading-indicator-message"></span>',"</div>","<svg/>",'<div class="stats-display"></div>'].join("")),this.$el.children().hide(),a&&this.renderData(a),this},showLoadingIndicator:function(a,b){a=a||"",b=b||"fast";var c=this.$el.find(".loading-indicator");a&&c.find(".loading-indicator-message").text(a),c.is(":visible")||(this.toggleStats(!1),c.css({left:this.config.width/2,top:this.config.height/2}).show())},hideLoadingIndicator:function(a){a=a||"fast",this.$el.find(".loading-indicator").hide()},renderData:function(a){this.$el.find(".controls").empty().append(this.renderControls(a)).show(),this.renderPlot(a),this.getStats(a)},renderControls:function(a){var b=this,c=$('<div class="left"></div>'),d=$('<div class="right"></div>');return c.append([this.renderPrevNext(a),this.renderPagination(a)]),d.append([this.renderLineInfo(a),$("<button>Stats</button>").addClass("stats-toggle-btn").click(function(){b.toggleStats()}),$("<button>Redraw</button>").addClass("rerender-btn").click(function(){b.renderPlot(a)})]),[c,d]},renderLineInfo:function(a){var b=this.dataset.metadata_data_lines||"an unknown number of",c=this.config.pagination.currPage*this.config.pagination.perPage,d=c+a.length;return $("<p/>").addClass("scatterplot-data-info").text(["Displaying lines",c+1,"to",d,"of",b,"lines"].join(" "))},renderPrevNext:function(a){function b(a){return $(['<li><a href="javascript:void(0);">',a,"</a></li>"].join(""))}if(!a||0===this.config.pagination.currPage&&a.length<this.config.pagination.perPage)return null;var c=this,d=this.dataset.metadata_data_lines,e=d?Math.ceil(d/this.config.pagination.perPage):void 0,f=b("Prev").click(function(){c.config.pagination.currPage>0&&(c.config.pagination.currPage-=1,c.fetchData())}),g=b("Next").click(function(){(!e||c.config.pagination.currPage<e-1)&&(c.config.pagination.currPage+=1,c.fetchData())}),h=$("<ul/>").addClass("pagination data-prev-next").append([f,g]);return 0===c.config.pagination.currPage&&f.addClass("disabled"),e&&c.config.pagination.currPage===e-1&&g.addClass("disabled"),h},renderPagination:function(a){function b(a){return $(['<li><a href="javascript:void(0);">',a,"</a></li>"].join(""))}function c(){d.config.pagination.currPage=$(this).data("page"),d.fetchData()}if(!a||0===this.config.pagination.currPage&&a.length<this.config.pagination.perPage)return null;for(var d=this,e=this.dataset.metadata_data_lines,f=e?Math.ceil(e/this.config.pagination.perPage):void 0,g=$("<ul/>").addClass("pagination data-pages"),h=0;f>h;h+=1){var i=b(h+1).attr("data-page",h).click(c);h===this.config.pagination.currPage&&i.addClass("active"),g.append(i)}return g},renderPlot:function(a){this.toggleStats(!1);var b=this.$el.find("svg");b.off().empty().show(),scatterplot(b.get(0),this.config,a)},getStats:function(a){var b=this;meanWorker=new Worker("/plugins/visualizations/scatterplot/static/worker-stats.js"),meanWorker.postMessage({data:a,keys:[this.config.xColumn,this.config.yColumn]}),meanWorker.onerror=function(){meanWorker.terminate()},meanWorker.onmessage=function(a){b.renderStats(a.data)}},renderStats:function(a){var b=this.$el.find(".stats-display"),c=this.config.x.label,d=this.config.y.label,e=$("<table/>").addClass("table").append(["<thead><th></th><th>",c,"</th><th>",d,"</th></thead>"].join("")).append(_.map(a,function(a,b){return $(["<tr><td>",b,"</td><td>",a[0],"</td><td>",a[1],"</td></tr>"].join(""))}));b.empty().append(e)},toggleStats:function(a){var b=this.$el.find(".stats-display");a=void 0===a?b.is(":hidden"):a,a?(this.$el.find("svg").hide(),b.show(),this.$el.find(".controls .stats-toggle-btn").text("Plot")):(b.hide(),this.$el.find("svg").show(),this.$el.find(".controls .stats-toggle-btn").text("Stats"))},toString:function(){return"ScatterplotView()"}}); \ No newline at end of file diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 config/plugins/visualizations/scatterplot/templates/scatterplot.mako --- a/config/plugins/visualizations/scatterplot/templates/scatterplot.mako +++ b/config/plugins/visualizations/scatterplot/templates/scatterplot.mako @@ -15,7 +15,6 @@ <script type="text/javascript" src="/static/scripts/libs/jquery/jquery.migrate.js"></script><script type="text/javascript" src="/static/scripts/libs/underscore.js"></script><script type="text/javascript" src="/static/scripts/libs/backbone/backbone.js"></script> -<script type="text/javascript" src="/static/scripts/libs/backbone/backbone-relational.js"></script><script type="text/javascript" src="/static/scripts/libs/handlebars.runtime.js"></script><script type="text/javascript" src="/static/scripts/libs/d3.js"></script><script type="text/javascript" src="/static/scripts/libs/bootstrap.js"></script> @@ -43,22 +42,18 @@ data = None ##data = list( hda.datatype.dataset_column_dataprovider( hda, limit=10000 ) ) %> - var hda = ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )}, - data = ${h.to_json_string( data )}, - querySettings = ${h.to_json_string( query_args )}, - config = _.extend( querySettings, { - containerSelector : '#chart', - dataset : hda, - }); - //console.debug( querySettings ); + var hda = ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )}; var editor = new ScatterplotConfigEditor({ el : $( '.scatterplot-editor' ).attr( 'id', 'scatterplot-editor-hda-' + hda.id ), - config : config + config : ${h.to_json_string( query_args )}, + dataset : ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )} }).render(); + window.editor = editor; // uncomment to auto render for development //$( '.render-button:visible' ).click(); }); + </script> %endif diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 doc/source/lib/galaxy.webapps.galaxy.api.rst --- a/doc/source/lib/galaxy.webapps.galaxy.api.rst +++ b/doc/source/lib/galaxy.webapps.galaxy.api.rst @@ -205,6 +205,11 @@ The request and response format should be considered alpha and are subject to change. +API Return Codes and Formats +================== + +A set of error codes for API requests is being established and will be +documented here. This is a long-term project however so stayed tuned. API Controllers =============== @@ -297,6 +302,14 @@ :undoc-members: :show-inheritance: +:mod:`lda_datasets` Module +-------------------------- + +.. automodule:: galaxy.webapps.galaxy.api.lda_datasets + :members: + :undoc-members: + :show-inheritance: + :mod:`libraries` Module ----------------------- @@ -393,3 +406,67 @@ :undoc-members: :show-inheritance: + +API Design Guidelines +===================== + +The following section outlines guidelines related to extending and/or modifing +the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by +many contributors and so clients SHOULD NOT expect the API will conform to +these guidelines - but developers contributing to the Galaxy API SHOULD follow +these guidelines. + + - API functionality should include docstring documentation for consumption + by readthedocs.org. + - Developers should familarize themselves with the HTTP status code definitions + http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses + should properly set the status code according to the result - in particular + 2XX responses should be used for successful requests, 4XX for various + kinds of client errors, and 5XX for errors on the server side. + - If there is an error processing some part of request (one item in a list + for instance), the status code should be set to reflect the error and the + partial result may or may not be returned depending on the controller - + this behavior should be documented. + - (TODO) API methods should throw a finite number of exceptions (defined in) + `galaxy.exceptions` and these should subclass `MessageException` and not + paste/wsgi HTTP exceptions. When possible, the framework itself should be + responsible catching these exceptions, setting the status code, and + building an error response. + - Error responses should not consist of plain text strings - they should be + dictionaries describing the error and containing the following keys (TODO: + spell out nature of this.) Various error conditions (once a format has + been chosen and framework to enforce it in place) should be spelled out + in this document. + - Backward compatibility is important and should maintained when possible. + If changing behavior in a non-backward compatibile way please ensure one + of the following holds - there is a strong reason to believe no consumers + depend on a behavior, the behavior is effectively broken, or the API + method being modified has not been part of a tagged dist release. + +The following bullet points represent good practices more than guidelines, please +consider them when modifying the API. + + - Functionality should not be copied and pasted between controllers - + consider refactoring functionality into associated classes or short of + that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance). + - API additions are more permanent changes to Galaxy than many other potential + changes and so a second opinion on API changes should be sought. (Consider a + pull request!) + - New API functionality should include functional tests. These functional + tests should be implemented in Python and placed in + `test/functional/api`. (Once such a framework is in place - it is not + right now). + - Changes to reflect modifications to the API should be pushed upstream to + the BioBlend project possible. + +Longer term goals/notes. + + - It would be advantageous to have a clearer separation of anonymous and + admin handling functionality. + - If at some point in the future, functionality needs to be added that + breaks backward compatibility in a significant way to a compontent used by + the community should be alerted - a "dev" variant of the API will be + established and the community should be alerted and given a timeframe + for when the old behavior will be replaced with the new behavior. + - Consistent standards for range-based requests, batch requests, filtered + requests, etc... should be established and documented here. diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 install_and_test_tool_shed_repositories.sh --- a/install_and_test_tool_shed_repositories.sh +++ b/install_and_test_tool_shed_repositories.sh @@ -2,14 +2,20 @@ # A good place to look for nose info: http://somethingaboutorange.com/mrl/projects/nose/ -# The test/install_and_test_tool_shed_repositories/functional_tests.py can not be executed directly, because it must have certain functional test definitions -# in sys.argv. Running it through this shell script is the best way to ensure that it has the required definitions. +# The test/install_and_test_tool_shed_repositories/functional_tests.py cannot be executed directly because it must +# have certain functional test definitions in sys.argv. Running it through this shell script is the best way to +# ensure that it has the required definitions. -# This script requires the following environment variables: +# This script requires setting of the following environment variables: # GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY - must be set to the API key for the tool shed that is being checked. # GALAXY_INSTALL_TEST_TOOL_SHED_URL - must be set to a URL that the tool shed is listening on. -# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to a tool sheds configuration file -# that does specify that url, otherwise repository installation will fail. + +# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to +# a tool sheds configuration file that does specify that url or repository installation will fail. + +# This script accepts the command line option -w to select which set of tests to run. The default behavior is to test +# first tool_dependency_definition repositories and then repositories with tools. Provide the value 'dependencies' +# to test only tool_dependency_definition repositories or 'tools' to test only repositories with tools. if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY ] ; then echo "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty." @@ -37,7 +43,45 @@ fi fi -python test/install_and_test_tool_shed_repositories/functional_tests.py $* -v --with-nosehtml --html-report-file \ - test/install_and_test_tool_shed_repositories/run_functional_tests.html \ - test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \ - test/functional/test_toolbox.py +test_tool_dependency_definitions () { + # Test installation of repositories of type tool_dependency_definition. + python test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py $* -v --with-nosehtml --html-report-file \ + test/install_and_test_tool_shed_repositories/tool_dependency_definitions/run_functional_tests.html \ + test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \ + test/functional/test_toolbox.py +} + +test_repositories_with_tools () { + # Test installation of repositories that contain valid tools with defined functional tests and a test-data directory containing test files. + python test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py $* -v --with-nosehtml --html-report-file \ + test/install_and_test_tool_shed_repositories/repositories_with_tools/run_functional_tests.html \ + test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \ + test/functional/test_toolbox.py +} + +which='both' + +while getopts "w:" arg; do + case $arg in + w) + which=$OPTARG + ;; + esac +done + +case $which in + # Use "-w tool_dependency_definitions" when you want to test repositories of type tool_dependency_definition. + tool_dependency_definitions) + test_tool_dependency_definitions + ;; + # Use "-w repositories_with_tools" parameter when you want to test repositories that contain tools. + repositories_with_tools) + test_repositories_with_tools + ;; + # No received parameters or any received parameter not in [ tool_dependency_definitions, repositories_with_tools ] + # will execute both scripts. + *) + test_tool_dependency_definitions + test_repositories_with_tools + ;; +esac diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -52,15 +52,44 @@ <param id="private_token">123456789changeme</param><!-- Uncomment the following statement to disable file staging (e.g. if there is a shared file system between Galaxy and the LWR - server). --> + server). Alternatively action can be set to 'copy' - to replace + http transfers with file system copies. --><!-- <param id="default_file_action">none</param> --> + <!-- The above option is just the default, the transfer behavior + none|copy|http can be configured on a per path basis via the + following file. See lib/galaxy/jobs/runners/lwr_client/action_mapper.py + for examples of how to configure this file. This is very beta + and nature of file will likely change. + --> + <!-- <param id="file_action_config">file_actions.json</param> --> + <!-- Uncomment following option to disable Galaxy tool dependency + resolution and utilize remote LWR's configuraiton of tool + dependency resolution instead (same options as Galaxy for + dependency resolution are available in LWR). + --> + <!-- <param id="dependency_resolution">remote</params> --> + <!-- Uncomment following option to enable setting metadata on remote + LWR server. The 'use_remote_datatypes' option is available for + determining whether to use remotely configured datatypes or local + ones (both alternatives are a little brittle). --> + <!-- <param id="remote_metadata">true</param> --> + <!-- <param id="use_remote_datatypes">false</param> --> + <!-- If remote LWR server is configured to run jobs as the real user, + uncomment the following line to pass the current Galaxy user + along. --> + <!-- <param id="submit_user">$__user_name__</param> --> + <!-- Various other submission parameters can be passed along to the LWR + whose use will depend on the remote LWR's configured job manager. + For instance: + --> + <!-- <param id="submit_native_specification">-P bignodes -R y -pe threads 8</param> --></destination><destination id="ssh_torque" runner="cli"><param id="shell_plugin">SecureShell</param><param id="job_plugin">Torque</param><param id="shell_username">foo</param><param id="shell_hostname">foo.example.org</param> - <param id="Job_Execution_Time">24:00:00</param> + <param id="job_Resource_List">walltime=24:00:00,ncpus=4</param></destination><destination id="condor" runner="condor"><!-- With no params, jobs are submitted to the 'vanilla' universe with: diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -1,16 +1,10 @@ from __future__ import absolute_import -import sys, os, atexit +import sys +import os -from galaxy import config, jobs, util, tools, web -import galaxy.tools.search -import galaxy.tools.data -import tool_shed.galaxy_install -import tool_shed.tool_shed_registry -from galaxy.web import security +from galaxy import config, jobs import galaxy.model -import galaxy.datatypes.registry import galaxy.security -from galaxy.objectstore import build_object_store_from_config import galaxy.quota from galaxy.tags.tag_handler import GalaxyTagHandler from galaxy.visualization.genomes import Genomes @@ -27,7 +21,8 @@ import logging log = logging.getLogger( __name__ ) -class UniverseApplication( object ): + +class UniverseApplication( object, config.ConfiguresGalaxyMixin ): """Encapsulates the state of a Universe application""" def __init__( self, **kwargs ): print >> sys.stderr, "python path is: " + ", ".join( sys.path ) @@ -38,72 +33,38 @@ self.config.check() config.configure_logging( self.config ) self.configure_fluent_log() - # Determine the database url - if self.config.database_connection: - db_url = self.config.database_connection - else: - db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database - # Set up the tool sheds registry - if os.path.isfile( self.config.tool_sheds_config ): - self.tool_shed_registry = tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) - else: - self.tool_shed_registry = None - # Initialize database / check for appropriate schema version. # If this - # is a new installation, we'll restrict the tool migration messaging. - from galaxy.model.migrate.check import create_or_verify_database - create_or_verify_database( db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options, app=self ) - # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed. - from tool_shed.galaxy_install.migrate.check import verify_tools - verify_tools( self, db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options ) - # Object store manager - self.object_store = build_object_store_from_config(self.config, fsmon=True) + + self._configure_tool_shed_registry() + + self._configure_object_store( fsmon=True ) + # Setup the database engine and ORM - from galaxy.model import mapping - self.model = mapping.init( self.config.file_path, - db_url, - self.config.database_engine_options, - database_query_profiling_proxy = self.config.database_query_profiling_proxy, - object_store = self.object_store, - trace_logger=self.trace_logger, - use_pbkdf2=self.config.get_bool( 'use_pbkdf2', True ) ) + config_file = kwargs.get( 'global_conf', {} ).get( '__file__', None ) + self._configure_models( check_migrate_databases=True, check_migrate_tools=True, config_file=config_file ) + # Manage installed tool shed repositories. - self.installed_repository_manager = tool_shed.galaxy_install.InstalledRepositoryManager( self ) - # Create an empty datatypes registry. - self.datatypes_registry = galaxy.datatypes.registry.Registry() - # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We - # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some - # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used. - # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype - # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict - # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence. - self.installed_repository_manager.load_proprietary_datatypes() - # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. - self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) + from tool_shed.galaxy_install import installed_repository_manager + self.installed_repository_manager = installed_repository_manager.InstalledRepositoryManager( self ) + + self._configure_datatypes_registry( self.installed_repository_manager ) galaxy.model.set_datatypes_registry( self.datatypes_registry ) + # Security helper - self.security = security.SecurityHelper( id_secret=self.config.id_secret ) + self._configure_security() # Tag handler self.tag_handler = GalaxyTagHandler() # Genomes self.genomes = Genomes( self ) # Data providers registry. self.data_provider_registry = DataProviderRegistry() - # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. - self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( tool_data_path=self.config.tool_data_path, - config_filename=self.config.tool_data_table_config_path ) - # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. - self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config, - tool_data_path=self.tool_data_tables.tool_data_path, - from_shed_config=False ) + + self._configure_tool_data_tables( from_shed_config=False ) + # Initialize the job management configuration self.job_config = jobs.JobConfiguration(self) - # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. - tool_configs = self.config.tool_configs - if self.config.migrated_tools_config not in tool_configs: - tool_configs.append( self.config.migrated_tools_config ) - self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) - # Search support for tools - self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) + + self._configure_toolbox() + # Load Data Manager self.data_managers = DataManagers( self ) # If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories. diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -1,6 +1,8 @@ """ Universe configuration builder. """ +# absolute_import needed for tool_shed package. +from __future__ import absolute_import import sys, os, tempfile, re import logging, logging.config @@ -33,15 +35,22 @@ self.umask = os.umask( 077 ) # get the current umask os.umask( self.umask ) # can't get w/o set, so set it back self.gid = os.getgid() # if running under newgrp(1) we'll need to fix the group of data created on the cluster + # Database related configuration self.database = resolve_path( kwargs.get( "database_file", "database/universe.sqlite" ), self.root ) self.database_connection = kwargs.get( "database_connection", False ) self.database_engine_options = get_database_engine_options( kwargs ) self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) ) self.database_query_profiling_proxy = string_as_bool( kwargs.get( "database_query_profiling_proxy", "False" ) ) + # Don't set this to true for production databases, but probably should # default to True for sqlite databases. self.database_auto_migrate = string_as_bool( kwargs.get( "database_auto_migrate", "False" ) ) + + # Install database related configuration (if different). + self.install_database_connection = kwargs.get( "install_database_connection", None ) + self.install_database_engine_options = get_database_engine_options( kwargs, model_prefix="install_" ) + # Where dataset files are stored self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root ) self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root ) @@ -439,7 +448,7 @@ admin_users = [ x.strip() for x in self.get( "admin_users", "" ).split( "," ) ] return ( user is not None and user.email in admin_users ) -def get_database_engine_options( kwargs ): +def get_database_engine_options( kwargs, model_prefix='' ): """ Allow options for the SQLAlchemy database engine to be passed by using the prefix "database_engine_option". @@ -455,7 +464,7 @@ 'pool_threadlocal': string_as_bool, 'server_side_cursors': string_as_bool } - prefix = "database_engine_option_" + prefix = "%sdatabase_engine_option_" % model_prefix prefix_len = len( prefix ) rval = {} for key, value in kwargs.iteritems(): @@ -466,6 +475,7 @@ rval[ key ] = value return rval + def configure_logging( config ): """ Allow some basic logging configuration to be read from ini file. @@ -506,3 +516,110 @@ sentry_handler.setLevel( logging.WARN ) root.addHandler( sentry_handler ) + +class ConfiguresGalaxyMixin: + """ Shared code for configuring Galaxy-like app objects. + """ + + def _configure_toolbox( self ): + # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. + tool_configs = self.config.tool_configs + if self.config.migrated_tools_config not in tool_configs: + tool_configs.append( self.config.migrated_tools_config ) + from galaxy import tools + self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) + # Search support for tools + import galaxy.tools.search + self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) + + def _configure_tool_data_tables( self, from_shed_config ): + from galaxy.tools.data import ToolDataTableManager + + # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. + self.tool_data_tables = ToolDataTableManager( tool_data_path=self.config.tool_data_path, + config_filename=self.config.tool_data_table_config_path ) + # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. + self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=from_shed_config ) + + def _configure_datatypes_registry( self, installed_repository_manager=None ): + from galaxy.datatypes import registry + # Create an empty datatypes registry. + self.datatypes_registry = registry.Registry() + if installed_repository_manager: + # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We + # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some + # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used. + # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype + # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict + # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence. + installed_repository_manager.load_proprietary_datatypes() + # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. + self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) + + def _configure_object_store( self, **kwds ): + from galaxy.objectstore import build_object_store_from_config + self.object_store = build_object_store_from_config( self.config, **kwds ) + + def _configure_security( self ): + from galaxy.web import security + self.security = security.SecurityHelper( id_secret=self.config.id_secret ) + + def _configure_tool_shed_registry( self ): + import tool_shed.tool_shed_registry + + # Set up the tool sheds registry + if os.path.isfile( self.config.tool_sheds_config ): + self.tool_shed_registry = tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) + else: + self.tool_shed_registry = None + + def _configure_models( self, check_migrate_databases=False, check_migrate_tools=False, config_file=None ): + """ + Preconditions: object_store must be set on self. + """ + if self.config.database_connection: + db_url = self.config.database_connection + else: + db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database + install_db_url = self.config.install_database_connection + # TODO: Consider more aggressive check here that this is not the same + # database file under the hood. + combined_install_database = not( install_db_url and install_db_url != db_url ) + install_db_url = install_db_url or db_url + + if check_migrate_databases: + # Initialize database / check for appropriate schema version. # If this + # is a new installation, we'll restrict the tool migration messaging. + from galaxy.model.migrate.check import create_or_verify_database + create_or_verify_database( db_url, config_file, self.config.database_engine_options, app=self ) + if not combined_install_database: + from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database + tsi_create_or_verify_database( install_db_url, self.config.install_database_engine_options, app=self ) + + if check_migrate_tools: + # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed. + from tool_shed.galaxy_install.migrate.check import verify_tools + verify_tools( self, install_db_url, config_file, self.config.database_engine_options ) + + from galaxy.model import mapping + self.model = mapping.init( self.config.file_path, + db_url, + self.config.database_engine_options, + map_install_models=combined_install_database, + database_query_profiling_proxy=self.config.database_query_profiling_proxy, + object_store=self.object_store, + trace_logger=getattr(self, "trace_logger", None), + use_pbkdf2=self.config.get_bool( 'use_pbkdf2', True ) ) + + if combined_install_database: + log.info("Install database targetting Galaxy's database configuration.") + self.install_model = self.model + else: + from galaxy.model.tool_shed_install import mapping as install_mapping + install_db_url = self.config.install_database_connection + log.info("Install database using its own connection %s" % install_db_url) + install_db_engine_options = self.config.install_database_engine_options + self.install_model = install_mapping.init( install_db_url, + install_db_engine_options ) diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/datatypes/checkers.py --- a/lib/galaxy/datatypes/checkers.py +++ b/lib/galaxy/datatypes/checkers.py @@ -2,6 +2,8 @@ from galaxy import util from StringIO import StringIO +HTML_CHECK_LINES = 100 + try: import Image as PIL except ImportError: @@ -32,9 +34,11 @@ regexp1 = re.compile( "<A\s+[^>]*HREF[^>]+>", re.I ) regexp2 = re.compile( "<IFRAME[^>]*>", re.I ) regexp3 = re.compile( "<FRAMESET[^>]*>", re.I ) - regexp4 = re.compile( "<META[^>]*>", re.I ) + regexp4 = re.compile( "<META[\W][^>]*>", re.I ) regexp5 = re.compile( "<SCRIPT[^>]*>", re.I ) lineno = 0 + # TODO: Potentially reading huge lines into string here, this should be + # reworked. for line in temp: lineno += 1 matches = regexp1.search( line ) or regexp2.search( line ) or regexp3.search( line ) or regexp4.search( line ) or regexp5.search( line ) @@ -42,7 +46,7 @@ if chunk is None: temp.close() return True - if lineno > 100: + if HTML_CHECK_LINES and (lineno > HTML_CHECK_LINES): break if chunk is None: temp.close() diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -199,6 +199,26 @@ out = "Can't create peek %s" % str( exc ) return out + def _archive_main_file(self, archive, display_name, data_filename): + """Called from _archive_composite_dataset to add central file to archive. + + Unless subclassed, this will add the main dataset file (argument data_filename) + to the archive, as an HTML file with its filename derived from the dataset name + (argument outfname). + + Returns a tuple of boolean, string, string: (error, msg, messagetype) + """ + error, msg, messagetype = False, "", "" + archname = '%s.html' % display_name # fake the real nature of the html file + try: + archive.add(data_filename, archname) + except IOError: + error = True + log.exception("Unable to add composite parent %s to temporary library download archive" % data_filename) + msg = "Unable to create archive for download, please report this error" + messagetype = "error" + return error, msg, messagetype + def _archive_composite_dataset( self, trans, data=None, **kwd ): # save a composite object into a compressed archive for downloading params = util.Params( kwd ) @@ -237,29 +257,27 @@ path = data.file_name fname = os.path.split(path)[-1] efp = data.extra_files_path - htmlname = os.path.splitext(outfname)[0] - if not htmlname.endswith(ext): - htmlname = '%s_%s' % (htmlname,ext) - archname = '%s.html' % htmlname # fake the real nature of the html file - try: - archive.add(data.file_name,archname) - except IOError: - error = True - log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name) - msg = "Unable to create archive for download, please report this error" - messagetype = 'error' - for root, dirs, files in os.walk(efp): - for fname in files: - fpath = os.path.join(root,fname) - rpath = os.path.relpath(fpath,efp) - try: - archive.add( fpath,rpath ) - except IOError: - error = True - log.exception( "Unable to add %s to temporary library download archive" % rpath) - msg = "Unable to create archive for download, please report this error" - messagetype = 'error' - continue + #Add any central file to the archive, + + display_name = os.path.splitext(outfname)[0] + if not display_name.endswith(ext): + display_name = '%s_%s' % (display_name, ext) + + error, msg, messagetype = self._archive_main_file(archive, display_name, path) + if not error: + #Add any child files to the archive, + for root, dirs, files in os.walk(efp): + for fname in files: + fpath = os.path.join(root,fname) + rpath = os.path.relpath(fpath,efp) + try: + archive.add( fpath,rpath ) + except IOError: + error = True + log.exception( "Unable to add %s to temporary library download archive" % rpath) + msg = "Unable to create archive for download, please report this error" + messagetype = 'error' + continue if not error: if params.do_action == 'zip': archive.close() @@ -288,7 +306,14 @@ return open( dataset.file_name ) def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd): - """ Old display method, for transition """ + """ Old display method, for transition - though still used by API and + test framework. Datatypes should be very careful if overridding this + method and this interface between datatypes and Galaxy will likely + change. + + TOOD: Document alternatives to overridding this method (data + providers?). + """ #Relocate all composite datatype display to a common location. composite_extensions = trans.app.datatypes_registry.get_composite_extensions( ) composite_extensions.append('html') # for archiving composite datatypes diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/datatypes/interval.py --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -389,7 +389,7 @@ MetadataElement( name="endCol", default=3, desc="End column", param=metadata.ColumnParameter ) MetadataElement( name="strandCol", desc="Strand column (click box & select)", param=metadata.ColumnParameter, optional=True, no_value=0 ) MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False ) - MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, multiple=True ) + MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, optional=True, multiple=True ) ###do we need to repeat these? they are the same as should be inherited from interval type def set_meta( self, dataset, overwrite = True, **kwd ): diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -348,7 +348,7 @@ try: aclass = getattr( module, datatype_class_name )() except Exception, e: - self.log.exception( 'Error calling method %s from class %s: %s' ( str( datatype_class_name ), str( module ), str( e ) ) ) + self.log.exception( 'Error calling method %s from class %s: %s', str( datatype_class_name ), str( module ), str( e ) ) ok = False if ok: if deactivate: @@ -598,6 +598,9 @@ tool_xml_text = """ <tool id="__SET_METADATA__" name="Set External Metadata" version="1.0.1" tool_type="set_metadata"><type class="SetMetadataTool" module="galaxy.tools"/> + <requirements> + <requirement type="package">samtools</requirement> + </requirements><action module="galaxy.tools.actions.metadata" class="SetMetadataToolAction"/><command>$__SET_EXTERNAL_METADATA_COMMAND_LINE__</command><inputs> diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -652,7 +652,7 @@ MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False ) MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) - MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True, visible=False ) + MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False ) MetadataElement( name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[] ) def sniff( self, filename ): diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/exceptions/__init__.py --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -1,6 +1,10 @@ """ Custom exceptions for Galaxy """ + +from galaxy import eggs +eggs.require( "Paste" ) + from paste import httpexceptions class MessageException( Exception ): diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -14,7 +14,6 @@ import shutil import subprocess import sys -import threading import traceback from galaxy import model, util from galaxy.datatypes import metadata @@ -39,21 +38,6 @@ # and should eventually become API'd TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' -class Sleeper( object ): - """ - Provides a 'sleep' method that sleeps for a number of seconds *unless* - the notify method is called (from a different thread). - """ - def __init__( self ): - self.condition = threading.Condition() - def sleep( self, seconds ): - self.condition.acquire() - self.condition.wait( seconds ) - self.condition.release() - def wake( self ): - self.condition.acquire() - self.condition.notify() - self.condition.release() class JobDestination( Bunch ): """ @@ -704,10 +688,7 @@ if self.command_line and self.command_line.startswith( 'python' ): self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root # Shell fragment to inject dependencies - if self.app.config.use_tool_dependencies: - self.dependency_shell_commands = self.tool.build_dependency_shell_commands() - else: - self.dependency_shell_commands = None + self.dependency_shell_commands = self.tool.build_dependency_shell_commands() # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished job.command_line = self.command_line @@ -1451,10 +1432,7 @@ if self.command_line and self.command_line.startswith( 'python' ): self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root # Shell fragment to inject dependencies - if self.app.config.use_tool_dependencies: - self.dependency_shell_commands = self.tool.build_dependency_shell_commands() - else: - self.dependency_shell_commands = None + self.dependency_shell_commands = self.tool.build_dependency_shell_commands() # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished task.command_line = self.command_line diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/actions/post.py --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -12,7 +12,7 @@ form = """ if (pja.action_type == "%s"){ p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + pja.output_name + "</span><div class='toolFormTitle'> %s <br/> on " + pja.output_name + "\ - <div style='float: right;' class='buttons'><img src='/static/images/delete_icon.png'></div></div><div class='toolFormBody'>"; + <div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>"; %s p_str += "</div><div class='toolParamHelp'>%s</div></div>"; }""" % (action_type, title, content, help) @@ -20,7 +20,7 @@ form = """ if (pja.action_type == "%s"){ p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + "</span><div class='toolFormTitle'> %s \ - <div style='float: right;' class='buttons'><img src='/static/images/delete_icon.png'></div></div><div class='toolFormBody'>"; + <div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>"; %s p_str += "</div><div class='toolParamHelp'>%s</div></div>"; }""" % (action_type, title, content, help) diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/command_factory.py --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -1,8 +1,11 @@ from os import getcwd from os.path import abspath +CAPTURE_RETURN_CODE = "return_code=$?" +YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' -def build_command( job, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): + +def build_command( runner, job_wrapper, include_metadata=False, include_work_dir_outputs=True, remote_command_params={} ): """ Compose the sequence of commands necessary to execute a job. This will currently include: @@ -13,64 +16,125 @@ - commands to set metadata (if include_metadata is True) """ - commands = job_wrapper.get_command_line() + commands_builder = CommandsBuilder(job_wrapper.get_command_line()) # All job runners currently handle this case which should never occur - if not commands: + if not commands_builder.commands: return None - # Remove trailing semi-colon so we can start hacking up this command. - # TODO: Refactor to compose a list and join with ';', would be more clean. - commands = commands.rstrip("; ") + __handle_version_command(commands_builder, job_wrapper) + __handle_task_splitting(commands_builder, job_wrapper) + __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params) + if include_work_dir_outputs: + __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params) + + if include_metadata and job_wrapper.requires_setting_metadata: + __handle_metadata(commands_builder, job_wrapper, runner, remote_command_params) + + return commands_builder.build() + + +def __handle_version_command(commands_builder, job_wrapper): # Prepend version string if job_wrapper.version_string_cmd: - commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands + version_command = "%s &> %s" % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands_builder.prepend_command(version_command) + +def __handle_task_splitting(commands_builder, job_wrapper): # prepend getting input files (if defined) - if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: - commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] ) + if getattr(job_wrapper, 'prepare_input_files_cmds', None): + commands_builder.prepend_commands(job_wrapper.prepare_input_files_cmds) + + +def __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params): + local_dependency_resolution = remote_command_params.get("dependency_resolution", "local") == "local" # Prepend dependency injection - if job_wrapper.dependency_shell_commands: - commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) + if job_wrapper.dependency_shell_commands and local_dependency_resolution: + commands_builder.prepend_commands(job_wrapper.dependency_shell_commands) - # Coping work dir outputs or setting metadata will mask return code of - # tool command. If these are used capture the return code and ensure - # the last thing that happens is an exit with return code. - capture_return_code_command = "; return_code=$?" - captured_return_code = False +def __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params): # Append commands to copy job outputs based on from_work_dir attribute. - if include_work_dir_outputs: - work_dir_outputs = job.get_work_dir_outputs( job_wrapper ) - if work_dir_outputs: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True + work_dir_outputs_kwds = {} + if 'working_directory' in remote_command_params: + work_dir_outputs_kwds['job_working_directory'] = remote_command_params['working_directory'] + work_dir_outputs = runner.get_work_dir_outputs( job_wrapper, **work_dir_outputs_kwds ) + if work_dir_outputs: + commands_builder.capture_return_code() + copy_commands = map(__copy_if_exists_command, work_dir_outputs) + commands_builder.append_commands(copy_commands) - commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" % - ( source_file, source_file, destination ) for ( source_file, destination ) in work_dir_outputs ] ) +def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_params): # Append metadata setting commands, we don't want to overwrite metadata # that was copied over in init_meta(), as per established behavior - if include_metadata and job_wrapper.requires_setting_metadata: - metadata_command = job_wrapper.setup_external_metadata( - exec_dir=abspath( getcwd() ), - tmp_dir=job_wrapper.working_directory, - dataset_files_path=job.app.model.Dataset.file_path, - output_fnames=job_wrapper.get_output_fnames(), - set_extension=False, - kwds={ 'overwrite' : False } - ) or '' - metadata_command = metadata_command.strip() - if metadata_command: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True - commands += "; cd %s; %s" % (abspath( getcwd() ), metadata_command) + metadata_kwds = remote_command_params.get('metadata_kwds', {}) + exec_dir = metadata_kwds.get( 'exec_dir', abspath( getcwd() ) ) + tmp_dir = metadata_kwds.get( 'tmp_dir', job_wrapper.working_directory ) + dataset_files_path = metadata_kwds.get( 'dataset_files_path', runner.app.model.Dataset.file_path ) + output_fnames = metadata_kwds.get( 'output_fnames', job_wrapper.get_output_fnames() ) + config_root = metadata_kwds.get( 'config_root', None ) + config_file = metadata_kwds.get( 'config_file', None ) + datatypes_config = metadata_kwds.get( 'datatypes_config', None ) + metadata_command = job_wrapper.setup_external_metadata( + exec_dir=exec_dir, + tmp_dir=tmp_dir, + dataset_files_path=dataset_files_path, + output_fnames=output_fnames, + set_extension=False, + config_root=config_root, + config_file=config_file, + datatypes_config=datatypes_config, + kwds={ 'overwrite' : False } + ) or '' + metadata_command = metadata_command.strip() + if metadata_command: + commands_builder.capture_return_code() + commands_builder.append_command("cd %s; %s" % (exec_dir, metadata_command)) - if captured_return_code: - commands += '; sh -c "exit $return_code"' - return commands +def __copy_if_exists_command(work_dir_output): + source_file, destination = work_dir_output + return "if [ -f %s ] ; then cp %s %s ; fi" % ( source_file, source_file, destination ) + + +class CommandsBuilder(object): + + def __init__(self, initial_command): + # Remove trailing semi-colon so we can start hacking up this command. + # TODO: Refactor to compose a list and join with ';', would be more clean. + commands = initial_command.rstrip("; ") + self.commands = commands + + # Coping work dir outputs or setting metadata will mask return code of + # tool command. If these are used capture the return code and ensure + # the last thing that happens is an exit with return code. + self.return_code_captured = False + + def prepend_command(self, command): + self.commands = "%s; %s" % (command, self.commands) + return self + + def prepend_commands(self, commands): + return self.prepend_command("; ".join(commands)) + + def append_command(self, command): + self.commands = "%s; %s" % (self.commands, command) + + def append_commands(self, commands): + self.append_command("; ".join(commands)) + + def capture_return_code(self): + if not self.return_code_captured: + self.return_code_captured = True + self.append_command(CAPTURE_RETURN_CODE) + + def build(self): + if self.return_code_captured: + self.append_command(YIELD_CAPTURED_CODE) + return self.commands + +__all__ = [build_command] diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -11,7 +11,8 @@ from sqlalchemy.sql.expression import and_, or_, select, func from galaxy import model -from galaxy.jobs import Sleeper, JobWrapper, TaskWrapper, JobDestination +from galaxy.util.sleeper import Sleeper +from galaxy.jobs import JobWrapper, TaskWrapper, JobDestination log = logging.getLogger( __name__ ) diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/manager.py --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -11,7 +11,8 @@ from Queue import Empty, Queue from galaxy import model -from galaxy.jobs import handler, JobWrapper, NoopQueue, Sleeper +from galaxy.util.sleeper import Sleeper +from galaxy.jobs import handler, JobWrapper, NoopQueue from galaxy.util.json import from_json_string log = logging.getLogger( __name__ ) diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -146,11 +146,13 @@ def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True ): return build_command( self, job_wrapper, include_metadata=include_metadata, include_work_dir_outputs=include_work_dir_outputs ) - def get_work_dir_outputs( self, job_wrapper ): + def get_work_dir_outputs( self, job_wrapper, job_working_directory=None ): """ Returns list of pairs (source_file, destination) describing path to work_dir output file and ultimate destination. """ + if not job_working_directory: + job_working_directory = os.path.abspath( job_wrapper.working_directory ) def in_directory( file, directory ): """ @@ -186,7 +188,7 @@ if hda_tool_output and hda_tool_output.from_work_dir: # Copy from working dir to HDA. # TODO: move instead of copy to save time? - source_file = os.path.join( os.path.abspath( job_wrapper.working_directory ), hda_tool_output.from_work_dir ) + source_file = os.path.join( job_working_directory, hda_tool_output.from_work_dir ) destination = job_wrapper.get_output_destination( output_paths[ dataset.dataset_id ] ) if in_directory( source_file, job_wrapper.working_directory ): output_pairs.append( ( source_file, destination ) ) @@ -196,7 +198,7 @@ log.exception( "from_work_dir specified a location not in the working directory: %s, %s" % ( source_file, job_wrapper.working_directory ) ) return output_pairs - def _handle_metadata_externally(self, job_wrapper): + def _handle_metadata_externally( self, job_wrapper, resolve_requirements=False ): """ Set metadata externally. Used by the local and lwr job runners where this shouldn't be attached to command-line to execute. @@ -210,6 +212,12 @@ tmp_dir=job_wrapper.working_directory, #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior kwds={ 'overwrite' : False } ) + if resolve_requirements: + dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands() + if dependency_shell_commands: + if isinstance( dependency_shell_commands, list ): + dependency_shell_commands = "&&".join( dependency_shell_commands ) + external_metadata_script = "%s&&%s" % ( dependency_shell_commands, external_metadata_script ) log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) ) external_metadata_proc = subprocess.Popen( args=external_metadata_script, shell=True, diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/cli_job/torque.py --- a/lib/galaxy/jobs/runners/cli_job/torque.py +++ b/lib/galaxy/jobs/runners/cli_job/torque.py @@ -35,7 +35,8 @@ echo $? > %s """ -argmap = { 'Execution_Time' : '-a', +argmap = { 'destination' : '-q', + 'Execution_Time' : '-a', 'Account_Name' : '-A', 'Checkpoint' : '-c', 'Error_Path' : '-e', diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -302,7 +302,15 @@ The external script will be run with sudo, and will setuid() to the specified user. Effectively, will QSUB as a different user (then the one used by Galaxy). """ - p = subprocess.Popen([ '/usr/bin/sudo', '-E', self.external_runJob_script, str(username), jobtemplate_filename ], + script_parts = self.external_runJob_script.split() + script = script_parts[0] + command = [ '/usr/bin/sudo', '-E', script] + for script_argument in script_parts[1:]: + command.append(script_argument) + + command.extend( [ str(username), jobtemplate_filename ] ) + log.info("Running command %s" % command) + p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = p.communicate() exitcode = p.returncode diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -110,7 +110,7 @@ job_wrapper.fail( "failure running job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return - self._handle_metadata_externally( job_wrapper ) + self._handle_metadata_externally( job_wrapper, resolve_requirements=True ) # Finish the job! try: job_wrapper.finish( stdout, stderr, exit_code ) diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -3,7 +3,9 @@ from galaxy import model from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner from galaxy.jobs import JobDestination +from galaxy.jobs.command_factory import build_command from galaxy.util import string_as_bool_or_none +from galaxy.util.bunch import Bunch import errno from time import sleep @@ -12,11 +14,15 @@ from .lwr_client import ClientManager, url_to_destination_params from .lwr_client import finish_job as lwr_finish_job from .lwr_client import submit_job as lwr_submit_job +from .lwr_client import ClientJobDescription log = logging.getLogger( __name__ ) __all__ = [ 'LwrJobRunner' ] +NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "LWR misconfiguration - LWR client configured to set metadata remotely, but remote LWR isn't properly configured with a galaxy_home directory." +NO_REMOTE_DATATYPES_CONFIG = "LWR client is configured to use remote datatypes configuration when setting metadata externally, but LWR is not configured with this information. Defaulting to datatypes_conf.xml." + class LwrJobRunner( AsynchronousJobRunner ): """ @@ -54,40 +60,31 @@ return job_state def queue_job(self, job_wrapper): - command_line = '' job_destination = job_wrapper.job_destination - try: - job_wrapper.prepare() - if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: - for cmd in job_wrapper.prepare_input_files_cmds: # run the commands to stage the input files - #log.debug( 'executing: %s' % cmd ) - if 0 != os.system(cmd): - raise Exception('Error running file staging command: %s' % cmd) - job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line - command_line = self.build_command_line( job_wrapper, include_metadata=False, include_work_dir_outputs=False ) - except: - job_wrapper.fail( "failure preparing job", exception=True ) - log.exception("failure running job %d" % job_wrapper.job_id) - return + command_line, client, remote_job_config = self.__prepare_job( job_wrapper, job_destination ) - # If we were able to get a command line, run the job if not command_line: - job_wrapper.finish( '', '' ) return try: - client = self.get_client_from_wrapper(job_wrapper) - output_files = self.get_output_files(job_wrapper) - input_files = job_wrapper.get_input_fnames() - working_directory = job_wrapper.working_directory - tool = job_wrapper.tool - config_files = job_wrapper.extra_filenames - job_id = lwr_submit_job(client, tool, command_line, config_files, input_files, output_files, working_directory) + dependency_resolution = LwrJobRunner.__dependency_resolution( client ) + remote_dependency_resolution = dependency_resolution == "remote" + requirements = job_wrapper.tool.requirements if remote_dependency_resolution else [] + client_job_description = ClientJobDescription( + command_line=command_line, + output_files=self.get_output_files(job_wrapper), + input_files=job_wrapper.get_input_fnames(), + working_directory=job_wrapper.working_directory, + tool=job_wrapper.tool, + config_files=job_wrapper.extra_filenames, + requirements=requirements, + ) + job_id = lwr_submit_job(client, client_job_description, remote_job_config) log.info("lwr job submitted with job_id %s" % job_id) job_wrapper.set_job_destination( job_destination, job_id ) job_wrapper.change_state( model.Job.states.QUEUED ) - except: + except Exception: job_wrapper.fail( "failure running job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return @@ -100,6 +97,52 @@ lwr_job_state.job_destination = job_destination self.monitor_job(lwr_job_state) + def __prepare_job(self, job_wrapper, job_destination): + """ Build command-line and LWR client for this job. """ + command_line = None + client = None + remote_job_config = None + try: + job_wrapper.prepare() + self.__prepare_input_files_locally(job_wrapper) + client = self.get_client_from_wrapper(job_wrapper) + tool = job_wrapper.tool + remote_job_config = client.setup(tool.id, tool.version) + remote_metadata = LwrJobRunner.__remote_metadata( client ) + remote_work_dir_copy = LwrJobRunner.__remote_work_dir_copy( client ) + dependency_resolution = LwrJobRunner.__dependency_resolution( client ) + metadata_kwds = self.__build_metadata_configuration(client, job_wrapper, remote_metadata, remote_job_config) + remote_command_params = dict( + working_directory=remote_job_config['working_directory'], + metadata_kwds=metadata_kwds, + dependency_resolution=dependency_resolution, + ) + command_line = build_command( + self, + job_wrapper=job_wrapper, + include_metadata=remote_metadata, + include_work_dir_outputs=remote_work_dir_copy, + remote_command_params=remote_command_params, + ) + except Exception: + job_wrapper.fail( "failure preparing job", exception=True ) + log.exception("failure running job %d" % job_wrapper.job_id) + + # If we were able to get a command line, run the job + if not command_line: + job_wrapper.finish( '', '' ) + + return command_line, client, remote_job_config + + def __prepare_input_files_locally(self, job_wrapper): + """Run task splitting commands locally.""" + prepare_input_files_cmds = getattr(job_wrapper, 'prepare_input_files_cmds', None) + if prepare_input_files_cmds is not None: + for cmd in prepare_input_files_cmds: # run the commands to stage the input files + if 0 != os.system(cmd): + raise Exception('Error running file staging command: %s' % cmd) + job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line + def get_output_files(self, job_wrapper): output_fnames = job_wrapper.get_output_fnames() return [ str( o ) for o in output_fnames ] @@ -130,34 +173,42 @@ run_results = client.raw_check_complete() stdout = run_results.get('stdout', '') stderr = run_results.get('stderr', '') - + working_directory_contents = run_results.get('working_directory_contents', []) # Use LWR client code to transfer/copy files back # and cleanup job if needed. completed_normally = \ job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] cleanup_job = self.app.config.cleanup_job - work_dir_outputs = self.get_work_dir_outputs( job_wrapper ) + remote_work_dir_copy = LwrJobRunner.__remote_work_dir_copy( client ) + if not remote_work_dir_copy: + work_dir_outputs = self.get_work_dir_outputs( job_wrapper ) + else: + # They have already been copied over to look like regular outputs remotely, + # no need to handle them differently here. + work_dir_outputs = [] output_files = self.get_output_files( job_wrapper ) finish_args = dict( client=client, working_directory=job_wrapper.working_directory, job_completed_normally=completed_normally, cleanup_job=cleanup_job, work_dir_outputs=work_dir_outputs, - output_files=output_files ) + output_files=output_files, + working_directory_contents=working_directory_contents ) failed = lwr_finish_job( **finish_args ) if failed: job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True) - except: + except Exception: message = "Failed to communicate with remote job server." job_wrapper.fail( message, exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return - self._handle_metadata_externally( job_wrapper ) + if not LwrJobRunner.__remote_metadata( client ): + self._handle_metadata_externally( job_wrapper, resolve_requirements=True ) # Finish the job try: job_wrapper.finish( stdout, stderr ) - except: + except Exception: log.exception("Job wrapper finish method failed") job_wrapper.fail("Unable to finish job", exception=True) @@ -225,3 +276,71 @@ job_state.old_state = True job_state.running = state == model.Job.states.RUNNING self.monitor_queue.put( job_state ) + + @staticmethod + def __dependency_resolution( lwr_client ): + dependency_resolution = lwr_client.destination_params.get( "dependency_resolution", "local" ) + if dependency_resolution not in ["none", "local", "remote"]: + raise Exception("Unknown dependency_resolution value encountered %s" % dependency_resolution) + return dependency_resolution + + @staticmethod + def __remote_metadata( lwr_client ): + remote_metadata = string_as_bool_or_none( lwr_client.destination_params.get( "remote_metadata", False ) ) + return remote_metadata + + @staticmethod + def __remote_work_dir_copy( lwr_client ): + # Right now remote metadata handling assumes from_work_dir outputs + # have been copied over before it runs. So do that remotely. This is + # not the default though because adding it to the command line is not + # cross-platform (no cp on Windows) and its un-needed work outside + # the context of metadata settting (just as easy to download from + # either place.) + return LwrJobRunner.__remote_metadata( lwr_client ) + + @staticmethod + def __use_remote_datatypes_conf( lwr_client ): + """ When setting remote metadata, use integrated datatypes from this + Galaxy instance or use the datatypes config configured via the remote + LWR. + + Both options are broken in different ways for same reason - datatypes + may not match. One can push the local datatypes config to the remote + server - but there is no guarentee these datatypes will be defined + there. Alternatively, one can use the remote datatype config - but + there is no guarentee that it will contain all the datatypes available + to this Galaxy. + """ + use_remote_datatypes = string_as_bool_or_none( lwr_client.destination_params.get( "use_remote_datatypes", False ) ) + return use_remote_datatypes + + def __build_metadata_configuration(self, client, job_wrapper, remote_metadata, remote_job_config): + metadata_kwds = {} + if remote_metadata: + remote_system_properties = remote_job_config.get("system_properties", {}) + remote_galaxy_home = remote_system_properties.get("galaxy_home", None) + if not remote_galaxy_home: + raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE) + metadata_kwds['exec_dir'] = remote_galaxy_home + outputs_directory = remote_job_config['outputs_directory'] + configs_directory = remote_job_config['configs_directory'] + outputs = [Bunch(false_path=os.path.join(outputs_directory, os.path.basename(path)), real_path=path) for path in self.get_output_files(job_wrapper)] + metadata_kwds['output_fnames'] = outputs + metadata_kwds['config_root'] = remote_galaxy_home + default_config_file = os.path.join(remote_galaxy_home, 'universe_wsgi.ini') + metadata_kwds['config_file'] = remote_system_properties.get('galaxy_config_file', default_config_file) + metadata_kwds['dataset_files_path'] = remote_system_properties.get('galaxy_dataset_files_path', None) + if LwrJobRunner.__use_remote_datatypes_conf( client ): + remote_datatypes_config = remote_system_properties.get('galaxy_datatypes_config_file', None) + if not remote_datatypes_config: + log.warn(NO_REMOTE_DATATYPES_CONFIG) + remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml') + metadata_kwds['datatypes_config'] = remote_datatypes_config + else: + integrates_datatypes_config = self.app.datatypes_registry.integrated_datatypes_configs + # Ensure this file gets pushed out to the remote config dir. + job_wrapper.extra_filenames.append(integrates_datatypes_config) + + metadata_kwds['datatypes_config'] = os.path.join(configs_directory, os.path.basename(integrates_datatypes_config)) + return metadata_kwds diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/lwr_client/__init__.py --- a/lib/galaxy/jobs/runners/lwr_client/__init__.py +++ b/lib/galaxy/jobs/runners/lwr_client/__init__.py @@ -6,9 +6,9 @@ """ -from .stager import submit_job, finish_job +from .stager import submit_job, finish_job, ClientJobDescription from .client import OutputNotFoundException from .manager import ClientManager from .destination import url_to_destination_params -__all__ = [ClientManager, OutputNotFoundException, url_to_destination_params, finish_job, submit_job] +__all__ = [ClientManager, OutputNotFoundException, url_to_destination_params, finish_job, submit_job, ClientJobDescription] diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/lwr_client/action_mapper.py --- a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py +++ b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py @@ -21,7 +21,7 @@ >>> from tempfile import NamedTemporaryFile >>> from os import unlink >>> f = NamedTemporaryFile(delete=False) - >>> f.write(json_string) + >>> write_result = f.write(json_string.encode('UTF-8')) >>> f.close() >>> class MockClient(): ... default_file_action = 'none' @@ -30,23 +30,23 @@ >>> mapper = FileActionMapper(MockClient()) >>> unlink(f.name) >>> # Test first config line above, implicit path prefix mapper - >>> mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input') - ('none',) + >>> mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input')[0] == u'none' + True >>> # Test another (2nd) mapper, this one with a different action - >>> mapper.action('/galaxy/data/files/000/dataset_1.dat', 'input') - ('transfer',) + >>> mapper.action('/galaxy/data/files/000/dataset_1.dat', 'input')[0] == u'transfer' + True >>> # Always at least copy work_dir outputs. - >>> mapper.action('/opt/galaxy/database/working_directory/45.sh', 'work_dir') - ('copy',) + >>> mapper.action('/opt/galaxy/database/working_directory/45.sh', 'work_dir')[0] == u'copy' + True >>> # Test glob mapper (matching test) - >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam', 'input') - ('copy',) + >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam', 'input')[0] == u'copy' + True >>> # Test glob mapper (non-matching test) - >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam.bai', 'input') - ('none',) + >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam.bai', 'input')[0] == u'none' + True >>> # Regex mapper test. - >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'input') - ('copy',) + >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'input')[0] == u'copy' + True """ def __init__(self, client): diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -50,7 +50,7 @@ return "No remote output found for path %s" % self.path -class Client(object): +class JobClient(object): """ Objects of this client class perform low-level communication with a remote LWR server. @@ -161,25 +161,23 @@ raise Exception("Unknown output_type returned from LWR server %s" % output_type) return output_path - def fetch_work_dir_output(self, source, working_directory, output_path, action='transfer'): + def fetch_work_dir_output(self, name, working_directory, output_path, action='transfer'): """ Download an output dataset specified with from_work_dir from the remote server. **Parameters** - source : str + name : str Path in job's working_directory to find output in. working_directory : str Local working_directory for the job. output_path : str Full path to output dataset. """ - output = open(output_path, "wb") - name = os.path.basename(source) if action == 'transfer': - self.__raw_download_output(name, self.job_id, "work_dir", output) - elif action == 'copy': + self.__raw_download_output(name, self.job_id, "work_dir", output_path) + else: # Even if action is none - LWR has a different work_dir so this needs to be copied. lwr_path = self._output_path(name, self.job_id, 'work_dir')['path'] self._copy(lwr_path, output_path) @@ -199,7 +197,7 @@ } self._raw_execute("download_output", output_params, output_path=output_path) - def launch(self, command_line): + def launch(self, command_line, requirements=[]): """ Run or queue up the execution of the supplied `command_line` on the remote server. @@ -213,6 +211,8 @@ submit_params = self._submit_params if submit_params: launch_params['params'] = dumps(submit_params) + if requirements: + launch_params['requirements'] = dumps([requirement.to_dict() for requirement in requirements]) return self._raw_execute("launch", launch_params) def kill(self): @@ -285,13 +285,13 @@ shutil.copyfile(source, destination) -class InputCachingClient(Client): +class InputCachingJobClient(JobClient): """ Beta client that cache's staged files to prevent duplication. """ def __init__(self, destination_params, job_id, job_manager_interface, client_cacher): - super(InputCachingClient, self).__init__(destination_params, job_id, job_manager_interface) + super(InputCachingJobClient, self).__init__(destination_params, job_id, job_manager_interface) self.client_cacher = client_cacher @parseJson() @@ -326,3 +326,55 @@ @parseJson() def file_available(self, path): return self._raw_execute("file_available", {"path": path}) + + +class ObjectStoreClient(object): + + def __init__(self, lwr_interface): + self.lwr_interface = lwr_interface + + @parseJson() + def exists(self, **kwds): + return self._raw_execute("object_store_exists", args=self.__data(**kwds)) + + @parseJson() + def file_ready(self, **kwds): + return self._raw_execute("object_store_file_ready", args=self.__data(**kwds)) + + @parseJson() + def create(self, **kwds): + return self._raw_execute("object_store_create", args=self.__data(**kwds)) + + @parseJson() + def empty(self, **kwds): + return self._raw_execute("object_store_empty", args=self.__data(**kwds)) + + @parseJson() + def size(self, **kwds): + return self._raw_execute("object_store_size", args=self.__data(**kwds)) + + @parseJson() + def delete(self, **kwds): + return self._raw_execute("object_store_delete", args=self.__data(**kwds)) + + @parseJson() + def get_data(self, **kwds): + return self._raw_execute("object_store_get_data", args=self.__data(**kwds)) + + @parseJson() + def get_filename(self, **kwds): + return self._raw_execute("object_store_get_filename", args=self.__data(**kwds)) + + @parseJson() + def update_from_file(self, **kwds): + return self._raw_execute("object_store_update_from_file", args=self.__data(**kwds)) + + @parseJson() + def get_store_usage_percent(self): + return self._raw_execute("object_store_get_store_usage_percent", args={}) + + def __data(self, **kwds): + return kwds + + def _raw_execute(self, command, args={}): + return self.lwr_interface.execute(command, args, data=None, input_path=None, output_path=None) diff -r 3871ca1bbfcfad27d63ca7ba8464714313a64ec0 -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 lib/galaxy/jobs/runners/lwr_client/destination.py --- a/lib/galaxy/jobs/runners/lwr_client/destination.py +++ b/lib/galaxy/jobs/runners/lwr_client/destination.py @@ -51,9 +51,10 @@ >>> destination_params = {"private_token": "12345", "submit_native_specification": "-q batch"} >>> result = submit_params(destination_params) - >>> result.items() - [('native_specification', '-q batch')] + >>> result + {'native_specification': '-q batch'} """ - return dict([(key[len(SUBMIT_PREFIX):], value) - for key, value in (destination_params or {}).iteritems() + destination_params = destination_params or {} + return dict([(key[len(SUBMIT_PREFIX):], destination_params[key]) + for key in destination_params if key.startswith(SUBMIT_PREFIX)]) This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/1bdc7d3beff4/ Changeset: 1bdc7d3beff4 Branch: search User: Kyle Ellrott Date: 2014-01-02 20:51:31 Summary: Fix to 'stringify' datetime object that was getting passed from to_dict LibraryFolder and into JSON converter. Affected #: 1 file diff -r 8388f17e522a775bfeb11f5a1d2eb2b3e2d80a60 -r 1bdc7d3beff411a31ac0656f648d433d3da04d69 lib/galaxy/model/item_attrs.py --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -3,6 +3,8 @@ # Cannot import galaxy.model b/c it creates a circular import graph. import galaxy import logging +import datetime + log = logging.getLogger( __name__ ) class RuntimeException( Exception ): @@ -182,6 +184,9 @@ except: if key in value_mapper: return value_mapper.get( key )( item ) + #If the item is of a class that needs to be 'stringified' before being put into a JSON data structure + if type(item) in [datetime.datetime]: + return str(item) return item # Create dict to represent item. https://bitbucket.org/galaxy/galaxy-central/commits/d05b1d649290/ Changeset: d05b1d649290 User: dannon Date: 2014-01-07 20:32:27 Summary: Merged in kellrott/galaxy-central/search (pull request #285) Make to_dict method stringify non-JSONable objects Affected #: 1 file diff -r abcc67f37be7f9450749fa87f6588e90964a6681 -r d05b1d649290a8563558e8721f4a1ca8336e8180 lib/galaxy/model/item_attrs.py --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -3,6 +3,8 @@ # Cannot import galaxy.model b/c it creates a circular import graph. import galaxy import logging +import datetime + log = logging.getLogger( __name__ ) class RuntimeException( Exception ): @@ -182,6 +184,9 @@ except: if key in value_mapper: return value_mapper.get( key )( item ) + #If the item is of a class that needs to be 'stringified' before being put into a JSON data structure + if type(item) in [datetime.datetime]: + return str(item) return item # Create dict to represent item. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.