1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5f0d3cbd97d9/ Changeset: 5f0d3cbd97d9 User: carlfeberhard Date: 2014-11-06 17:56:13+00:00 Summary: History structure: incorporate tool data, refactor; HDA API: return create_time Affected #: 30 files diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff client/galaxy/scripts/mvc/history/history-panel-edit-current.js --- a/client/galaxy/scripts/mvc/history/history-panel-edit-current.js +++ b/client/galaxy/scripts/mvc/history/history-panel-edit-current.js @@ -164,12 +164,13 @@ _setUpCollectionListeners : function(){ _super.prototype._setUpCollectionListeners.call( this ); + //TODO:?? may not be needed? see history-panel-edit, 369 // if a hidden item is created (gen. by a workflow), moves thru the updater to the ready state, // then: remove it from the collection if the panel is set to NOT show hidden datasets this.collection.on( 'state:ready', function( model, newState, oldState ){ if( ( !model.get( 'visible' ) ) && ( !this.storage.get( 'show_hidden' ) ) ){ - this.removeItemView( this.viewFromModel( model ) ); + this.removeItemView( model ); } }, this ); }, diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff client/galaxy/scripts/mvc/history/history-structure-view.js --- a/client/galaxy/scripts/mvc/history/history-structure-view.js +++ b/client/galaxy/scripts/mvc/history/history-structure-view.js @@ -1,79 +1,184 @@ define([ + 'mvc/history/job-dag', 'mvc/job/job-model', 'mvc/job/job-li', 'mvc/history/history-content-model', - 'mvc/history/job-dag', + 'mvc/dataset/dataset-li', 'mvc/base-mvc', 'utils/localization', 'libs/d3' -], function( JOB, JOB_LI, HISTORY_CONTENT, JobDAG, BASE_MVC, _l ){ +], function( JobDAG, JOB, JOB_LI, HISTORY_CONTENT, DATASET_LI, BASE_MVC, _l ){ // ============================================================================ -/* TODO: -change component = this to something else +/* +TODO: + disruptive: + handle collections + retain contents to job relationships (out/input name) + + display when *only* copied datasets + need to change when/how joblessVertices are created + + components should be full height containers that scroll individually + + use history contents views for job outputCollection, not vanilla datasets + need hid + + show datasets when job not expanded + make them external to the job display + connect jobs by dataset + which datasets from job X are which inputs in job Y? + + make job data human readable (needs tool data) + show only tool.inputs with labels (w/ job.params as values) + input datasets are special + they don't appear in job.params + have to connect to datasets in the dag + connect job.inputs to any tool.inputs by tool.input.name (in params) + +API: seems like this could be handled there - duplicating the input data in the proper param space + + collections + + use cases: + operations by thread: + copy to new history + rerun + to workflow + operations by branch (all descendants): + copy to new history + rerun + to workflow + signal to noise: + collapse/expand branch + hide jobs + visually isolate branch (hide other jobs) of thread + zoom (somehow) + + layout changes: + move branch to new column in component + complicated + pyramid + circular + sources on inner radius + expansion in vertical: + obscures relations due to height + could move details to side panel + difficult to compare two+ jobs/datasets when at different points in the topo + + (other) controls: + (optionally) filter all deleted + (optionally) filter all hidden + //(optionally) filter __SET_METADATA__ + //(optionally) filter error'd jobs + help and explanation + filtering/searching of jobs + + challenges: + difficult to scale dom (for zoomout) + possible to use css transforms? + transform svg and dom elements + it is possible to use css transforms on svg nodes + use transform-origin to select origin to top left + on larger histories the svg section may become extremely large due to distance from output to input + + how-to: + descendant ids: _.keys( component.depth/breadthFirstSearchTree( start ).vertices ) + + in-panel view of anc desc + */ // ============================================================================ /** * */ +window.JobDAG = JobDAG; var HistoryStructureComponent = Backbone.View.extend( BASE_MVC.LoggableMixin ).extend({ //logger : console, className : 'history-structure-component', + _INITIAL_ZOOM_LEVEL : 1.0, + _MIN_ZOOM_LEVEL : 0.25, + _LINK_ID_SEP : '-to-', + _VERTEX_NAME_DATA_KEY : 'vertex-name', + + JobItemClass : JOB_LI.JobListItemView, + ContentItemClass : DATASET_LI.DatasetListItemView, + initialize : function( attributes ){ this.log( this + '(HistoryStructureComponent).initialize:', attributes ); this.component = attributes.component; - this._jobLiMap = {}; - this._createJobModels(); + this._liMap = {}; + this._createVertexItems(); + + this.zoomLevel = attributes.zoomLevel || this._INITIAL_ZOOM_LEVEL; this.layout = this._createLayout( attributes.layoutOptions ); }, - _createJobModels : function(){ + _createVertexItems : function(){ var view = this; view.component.eachVertex( function( vertex ){ - var jobJSON = vertex.data.job, - job = new JOB.Job( jobJSON ); - - // get the models of the outputs for this job from the history - var outputModels = _.map( job.get( 'outputs' ), function( output ){ - //note: output is { src: 'hda/dataset_collection', id: <some id> } - // job output doesn't *quite* match up to normal typeId - var type = output.src === 'hda'? 'dataset' : 'dataset_collection', - typeId = HISTORY_CONTENT.typeIdStr( type, output.id ); - return view.model.contents.get( typeId ); - }); - // set the collection (HistoryContents) for the job to that json (setting historyId for proper ajax urls) - job.outputCollection.reset( outputModels ); - job.outputCollection.historyId = view.model.id; - //this.debug( job.outputCollection ); - - // create the bbone view for the job (to be positioned later accrd. to the layout) and cache - var li = new JOB_LI.JobListItemView({ model: job }); - //var li = new JOB_LI.JobListItemView({ model: job }); - li.render( 0 ).$el.appendTo( view.$el ); - view._jobLiMap[ job.id ] = li; - view._setUpJobListeners( li ); - +//TODO: hack + var type = vertex.data.job? 'job' : 'copy', + li; + if( type === 'job' ){ + li = view._createJobListItem( vertex ); + } else if( type === 'copy' ){ + li = view._createContentListItem( vertex ); + } + view._liMap[ vertex.name ] = li; }); - return view.jobs; + view.debug( '_liMap:', view._liMap ); }, - _setUpJobListeners : function( jobLi ){ - // update the layout during expansion and collapsing of job and output li's - jobLi.on( 'expanding expanded collapsing collapsed', this.render, this ); - jobLi.foldout.on( 'view:expanding view:expanded view:collapsing view:collapsed', this.render, this ); + _createJobListItem : function( vertex ){ + this.debug( '_createJobListItem:', vertex ); + var view = this, + jobData = vertex.data, + job = new JOB.Job( jobData.job ); + + // get the models of the outputs for this job from the history + var outputModels = _.map( job.get( 'outputs' ), function( output ){ + //note: output is { src: 'hda/dataset_collection', id: <some id> } + // job output doesn't *quite* match up to normal typeId + var type = output.src === 'hda'? 'dataset' : 'dataset_collection', + typeId = HISTORY_CONTENT.typeIdStr( type, output.id ); + return view.model.contents.get( typeId ); + }); + // set the collection (HistoryContents) for the job to that json (setting historyId for proper ajax urls) + job.outputCollection.reset( outputModels ); + job.outputCollection.historyId = view.model.id; + //this.debug( job.outputCollection ); + + // create the bbone view for the job (to be positioned later accrd. to the layout) and cache + var li = new view.JobItemClass({ model: job, tool: jobData.tool, jobData: jobData }); + li.on( 'expanding expanded collapsing collapsed', view.renderGraph, view ); + li.foldout.on( 'view:expanding view:expanded view:collapsing view:collapsed', view.renderGraph, view ); + return li; + }, + + _createContentListItem : function( vertex ){ + this.debug( '_createContentListItem:', vertex ); + var view = this, + content = vertex.data, + typeId = HISTORY_CONTENT.typeIdStr( content.history_content_type, content.id ); + content = view.model.contents.get( typeId ); + var li = new view.ContentItemClass({ model: content }); + li.on( 'expanding expanded collapsing collapsed', view.renderGraph, view ); + return li; }, layoutDefaults : { - paddingTop : 8, - paddingLeft : 20, linkSpacing : 16, - jobHeight : 308, - jobWidthSpacing : 320, + linkWidth : 0, + linkHeight : 0, + jobWidth : 300, + jobHeight : 300, + jobSpacing : 12, linkAdjX : 4, linkAdjY : 0 }, @@ -85,8 +190,7 @@ layout = _.extend( options, { nodeMap : {}, links : [], - el : { width: 0, height: 0 }, - svg : { width: 0, height: 0, top: 0, left: 0 } + svg : { width: 0, height: 0 } }); vertices.forEach( function( v, j ){ @@ -105,56 +209,50 @@ return layout; }, - _updateLayout : function(){ - var view = this, - layout = view.layout; - - layout.svg.height = layout.paddingTop + ( layout.linkSpacing * _.size( layout.nodeMap ) ); - layout.el.height = layout.svg.height + layout.jobHeight; - -//TODO:?? can't we just alter the component v and e's directly? - // layout the job views putting jobWidthSpacing btwn each - var x = layout.paddingLeft, - y = layout.svg.height; - _.each( layout.nodeMap, function( node, jobId ){ - //this.debug( node, jobId ); - node.x = x; - node.y = y; - x += layout.jobWidthSpacing; - }); - layout.el.width = layout.svg.width = Math.max( layout.el.width, x ); - - // layout the links - connecting each job by it's main coords (currently) -//TODO: somehow adjust the svg height based on the largest distance the longest connection needs - layout.links.forEach( function( link ){ - var source = layout.nodeMap[ link.source ], - target = layout.nodeMap[ link.target ]; - link.x1 = source.x + layout.linkAdjX; - link.y1 = source.y + layout.linkAdjY; - link.x2 = target.x + layout.linkAdjX; - link.y2 = target.y + layout.linkAdjY; - }); - //this.debug( JSON.stringify( layout, null, ' ' ) ); - return this.layout; - }, - render : function( options ){ this.debug( this + '.render:', options ); var view = this; + view.$el.html([ + '<header></header>', + '<nav class="controls"></nav>', + '<figure class="graph"></figure>', + '<footer></footer>' + ].join( '' ) ); + + var $graph = view.$graph(); + view.component.eachVertex( function( vertex ){ + view._liMap[ vertex.name ].render( 0 ).$el.appendTo( $graph ) + // store the name in the DOM and cache by that name + .data( view._VERTEX_NAME_DATA_KEY, vertex.name ); + }); + view.renderGraph(); + return this; + }, + + $graph : function(){ + return this.$( '.graph' ); + }, + + renderGraph : function( options ){ + this.debug( this + '.renderGraph:', options ); + var view = this; function _render(){ + view._updateLayout(); // set up the display containers - view.$el - .width( view.layout.el.width ) - .height( view.layout.el.height ); + view.$graph() + // use css3 transform to scale component graph + .css( 'transform', [ 'scale(', view.zoomLevel, ',', view.zoomLevel, ')' ].join( '' ) ) + .width( view.layout.svg.width ) + .height( view.layout.svg.height ); view.renderSVG(); // position the job views accrd. to the layout view.component.eachVertex( function( v ){ -//TODO:? liMap needed - can't we attach to vertex? - var li = view._jobLiMap[ v.name ], - position = view.layout.nodeMap[ li.model.id ]; +//TODO:?? liMap needed - can't we attach to vertex? + var li = view._liMap[ v.name ], + position = view.layout.nodeMap[ v.name ]; //this.debug( position ); li.$el.css({ top: position.y, left: position.x }); }); @@ -168,13 +266,52 @@ return this; }, - renderSVG : function(){ + _updateLayout : function(){ + this.debug( this + '._updateLayout:' ); var view = this, layout = view.layout; - var svg = d3.select( this.el ).select( 'svg' ); + layout.linkHeight = layout.linkSpacing * _.size( layout.nodeMap ); + layout.svg.height = layout.linkHeight + layout.jobHeight; + + // reset for later max comparison + layout.svg.width = 0; + +//TODO:?? can't we just alter the component v and e's directly? + // layout the job views putting jobSpacing btwn each + var x = 0, + y = layout.linkHeight; + _.each( layout.nodeMap, function( node, jobId ){ + //this.debug( node, jobId ); + node.x = x; + node.y = y; + x += layout.jobWidth + layout.jobSpacing; + }); + layout.svg.width = layout.linkWidth = Math.max( layout.svg.width, x ); + + // layout the links - connecting each job by it's main coords (currently) +//TODO: somehow adjust the svg height based on the largest distance the longest connection needs + layout.links.forEach( function( link ){ + var source = layout.nodeMap[ link.source ], + target = layout.nodeMap[ link.target ]; + link.x1 = source.x + layout.linkAdjX; + link.y1 = source.y + layout.linkAdjY; + link.x2 = target.x + layout.linkAdjX; + link.y2 = target.y + layout.linkAdjY; + }); + + this.debug( JSON.stringify( layout, null, ' ' ) ); + return this.layout; + }, + + renderSVG : function(){ + this.debug( this + '.renderSVG:' ); + var view = this, + layout = view.layout; + + var svg = d3.select( this.$graph().get(0) ).select( 'svg' ); if( svg.empty() ){ - svg = d3.select( this.el ).append( 'svg' ); + svg = d3.select( this.$graph().get(0) ).append( 'svg' ); } svg @@ -183,14 +320,14 @@ function highlightConnect( d ){ d3.select( this ).classed( 'highlighted', true ); - view._jobLiMap[ d.source ].$el.addClass( 'highlighted' ); - view._jobLiMap[ d.target ].$el.addClass( 'highlighted' ); + view._liMap[ d.source ].$el.addClass( 'highlighted' ); + view._liMap[ d.target ].$el.addClass( 'highlighted' ); } function unhighlightConnect( d ){ d3.select( this ).classed( 'highlighted', false ); - view._jobLiMap[ d.source ].$el.removeClass( 'highlighted' ); - view._jobLiMap[ d.target ].$el.removeClass( 'highlighted' ); + view._liMap[ d.source ].$el.removeClass( 'highlighted' ); + view._liMap[ d.target ].$el.removeClass( 'highlighted' ); } var connections = svg.selectAll( '.connection' ) @@ -199,21 +336,19 @@ connections .enter().append( 'path' ) .attr( 'class', 'connection' ) - .attr( 'id', function( d ){ return d.source + '-' + d.target; }) + .attr( 'id', function( d ){ return [ d.source, d.target ].join( view._LINK_ID_SEP ); }) .on( 'mouseover', highlightConnect ) .on( 'mouseout', unhighlightConnect ); connections .attr( 'd', function( d ){ return view._connectionPath( d ); }); -//TODO: ? can we use tick here to update the links? - return svg.node(); }, _connectionPath : function( d ){ var CURVE_X = 0, - controlY = ( ( d.x2 - d.x1 ) / this.layout.svg.width ) * this.layout.svg.height; + controlY = ( ( d.x2 - d.x1 ) / this.layout.svg.width ) * this.layout.linkHeight; return [ 'M', d.x1, ',', d.y1, ' ', 'C', @@ -224,11 +359,12 @@ }, events : { - 'mouseover .job.list-item' : function( ev ){ this.highlightConnected( ev.currentTarget, true ); }, - 'mouseout .job.list-item' : function( ev ){ this.highlightConnected( ev.currentTarget, false ); } + 'mouseover .graph > .list-item' : function( ev ){ this.highlightConnected( ev.currentTarget, true ); }, + 'mouseout .graph > .list-item' : function( ev ){ this.highlightConnected( ev.currentTarget, false ); } }, highlightConnected : function( jobElement, highlight ){ + this.debug( 'highlightConnected', jobElement, highlight ); highlight = highlight !== undefined? highlight : true; var view = this, @@ -237,21 +373,32 @@ connectionClass = highlight? 'connection highlighted' : 'connection'; //console.debug( 'mouseover', this ); - var $jobs = jobClassFn.call( $( jobElement ), 'highlighted' ), - id = $jobs.attr( 'id' ).replace( 'job-', '' ); + var $hoverTarget = jobClassFn.call( $( jobElement ), 'highlighted' ), + id = $hoverTarget.data( view._VERTEX_NAME_DATA_KEY ); // immed. ancestors component.edges({ target: id }).forEach( function( edge ){ - jobClassFn.call( view.$( '#job-' + edge.source ), 'highlighted' ); - view.$( '#' + edge.source + '-' + id ).attr( 'class', connectionClass ); + var ancestorId = edge.source, + ancestorLi = view._liMap[ ancestorId ]; + //view.debug( '\t ancestor:', ancestorId, ancestorLi ); + jobClassFn.call( ancestorLi.$el, 'highlighted' ); + view.$( '#' + ancestorId + view._LINK_ID_SEP + id ).attr( 'class', connectionClass ); }); // descendants component.vertices[ id ].eachEdge( function( edge ){ - jobClassFn.call( view.$( '#job-' + edge.target ), 'highlighted' ); - view.$( '#' + id + '-' + edge.target ).attr( 'class', connectionClass ); + var descendantId = edge.target, + descendantLi = view._liMap[ descendantId ]; + //view.debug( '\t descendant:', descendantId, descendantLi ); + jobClassFn.call( descendantLi.$el, 'highlighted' ); + view.$( '#' + id + view._LINK_ID_SEP + descendantId ).attr( 'class', connectionClass ); }); }, + zoom : function( level ){ + this.zoomLevel = Math.min( 1.0, Math.max( this._MIN_ZOOM_LEVEL, level ) ); + return this.renderGraph(); + }, + toString : function(){ return 'HistoryStructureComponent(' + this.model.id + ')'; } @@ -268,45 +415,34 @@ className : HistoryStructureComponent.prototype.className + ' vertical', - layoutDefaults : { - paddingTop : 8, - paddingLeft : 20, - linkSpacing : 16, - jobWidth : 308, - jobHeight : 308, - initialSpacing : 64, - jobSpacing : 16, + layoutDefaults : _.extend( _.clone( HistoryStructureComponent.prototype.layoutDefaults ), { linkAdjX : 0, linkAdjY : 4 - }, + }), //TODO: how can we use the dom height of the job li's - they're not visible when this is called? _updateLayout : function(){ + this.debug( this + '._updateLayout:' ); var view = this, layout = view.layout; //this.info( this.cid, '_updateLayout' ) - layout.svg.width = layout.paddingLeft + ( layout.linkSpacing * _.size( layout.nodeMap ) ); - layout.el.width = layout.svg.width + layout.jobWidth; + layout.linkWidth = layout.linkSpacing * _.size( layout.nodeMap ); + layout.svg.width = layout.linkWidth + layout.jobWidth; // reset height - we'll get the max Y below to assign to it - layout.el.height = 0; + layout.svg.height = 0; -//TODO:?? can't we just alter the component v and e's directly? - var x = layout.svg.width, - y = layout.paddingTop; - _.each( layout.nodeMap, function( node, jobId ){ - //this.debug( node, jobId ); + //TODO:?? can't we just alter the component v and e's directly? + var x = layout.linkWidth, + y = 0; + _.each( layout.nodeMap, function( node, nodeId ){ node.x = x; node.y = y; - var li = view._jobLiMap[ jobId ]; - if( li.$el.is( ':visible' ) ){ - y += li.$el.height() + layout.jobSpacing; - } else { - y += layout.initialSpacing + layout.jobSpacing; - } + var li = view._liMap[ nodeId ]; + y += li.$el.height() + layout.jobSpacing; }); - layout.el.height = layout.svg.height = Math.max( layout.el.height, y ); + layout.linkHeight = layout.svg.height = Math.max( layout.svg.height, y ); // layout the links - connecting each job by it's main coords (currently) layout.links.forEach( function( link ){ @@ -316,17 +452,16 @@ link.y1 = source.y + layout.linkAdjY; link.x2 = target.x + layout.linkAdjX; link.y2 = target.y + layout.linkAdjY; - view.debug( 'link:', link.x1, link.y1, link.x2, link.y2, link ); + //view.debug( 'link:', link.x1, link.y1, link.x2, link.y2, link ); }); - //this.debug( JSON.stringify( layout, null, ' ' ) ); - view.debug( 'el:', layout.el ); - view.debug( 'svg:', layout.svg ); + + this.debug( JSON.stringify( layout, null, ' ' ) ); return layout; }, _connectionPath : function( d ){ var CURVE_Y = 0, - controlX = ( ( d.y2 - d.y1 ) / this.layout.svg.height ) * this.layout.svg.width; + controlX = ( ( d.y2 - d.y1 ) / this.layout.svg.height ) * this.layout.linkWidth; return [ 'M', d.x1, ',', d.y1, ' ', 'C', @@ -352,40 +487,60 @@ className : 'history-structure', + _layoutToComponentClass : { + 'horizontal' : HistoryStructureComponent, + 'vertical' : VerticalHistoryStructureComponent + }, + //_DEFAULT_LAYOUT : 'horizontal', + _DEFAULT_LAYOUT : 'vertical', + initialize : function( attributes ){ + this.layout = _.contains( attributes.layout, _.keys( this._layoutToComponentClass ) )? + attributes.layout : this._DEFAULT_LAYOUT; this.log( this + '(HistoryStructureView).initialize:', attributes, this.model ); -//TODO: to model - this.jobs = attributes.jobs; + //TODO:?? to model - maybe glom jobs onto model in order to persist + // cache jobs since we need to re-create the DAG if settings change + this._processTools( attributes.tools ); + this._processJobs( attributes.jobs ); this._createDAG(); }, + _processTools : function( tools ){ + this.tools = tools || {}; + return this.tools; + }, + + _processJobs : function( jobs ){ + this.jobs = jobs || []; + return this.jobs; + }, + _createDAG : function(){ this.dag = new JobDAG({ historyContents : this.model.contents.toJSON(), + tools : this.tools, jobs : this.jobs, excludeSetMetadata : true, excludeErroredJobs : true }); -//window.dag = this.dag; - this.log( this + '.dag:', this.dag ); - + this.debug( this + '.dag:', this.dag ); this._createComponents(); }, _createComponents : function(){ this.log( this + '._createComponents' ); var structure = this; -//window.structure = structure; structure.componentViews = structure.dag.weakComponentGraphArray().map( function( componentGraph ){ return structure._createComponent( componentGraph ); }); + return structure.componentViews; }, _createComponent : function( component ){ this.log( this + '._createComponent:', component ); - //return new HistoryStructureComponent({ - return new VerticalHistoryStructureComponent({ + var ComponentClass = this._layoutToComponentClass[ this.layout ]; + return new ComponentClass({ model : this.model, component : component }); @@ -395,7 +550,7 @@ this.log( this + '.render:', options ); var structure = this; - structure.$el.html([ + structure.$el.addClass( 'clear' ).html([ '<div class="controls"></div>', '<div class="components"></div>' ].join( '' )); @@ -410,8 +565,17 @@ return this.$( '.components' ); }, + changeLayout : function( layout ){ + if( !( layout in this._layoutToComponentClass ) ){ + throw new Error( this + ': unknown layout: ' + layout ); + } + this.layout = layout; + this._createComponents(); + return this.render(); + }, + toString : function(){ - return 'HistoryStructureView(' + ')'; + return 'HistoryStructureView(' + this.model.id + ')'; } }); diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff client/galaxy/scripts/mvc/history/job-dag.js --- a/client/galaxy/scripts/mvc/history/job-dag.js +++ b/client/galaxy/scripts/mvc/history/job-dag.js @@ -9,15 +9,19 @@ * using the connections between job inputs and outputs. */ var JobDAG = function( options ){ + options = options || {}; var self = this; //this.logger = console; + self.filters = []; + // instance vars //TODO: needed? + self._jobsData = []; self._historyContentsMap = {}; - self.filters = []; + self._toolMap = {}; - self._idMap = {}; + self._outputIdToJobMap = {}; self.noInputJobs = []; self.noOutputJobs = []; @@ -25,7 +29,11 @@ self.filteredSetMetadata = []; self.filteredErroredJobs = []; - _super.call( self, true, null, options ); + self.dataKeys = [ 'jobs', 'historyContents', 'tools' ]; + _super.call( self, true, + _.pick( options, self.dataKeys ), + _.omit( options, self.dataKeys ) + ); }; JobDAG.prototype = new GRAPH.Graph(); JobDAG.prototype.constructor = JobDAG; @@ -33,29 +41,19 @@ // add logging ability - turn off/on using the this.logger statement above addLogging( JobDAG ); + // ---------------------------------------------------------------------------- /** process jobs, options, filters, and any history data, then create the graph */ JobDAG.prototype.init = function _init( options ){ options = options || {}; - _super.prototype.init.call( this, options ); - var self = this, - historyContentsJSON = options.historyContents || []; - jobsJSON = options.jobs || []; - - historyContentsJSON.forEach( function( content, i ){ - self._historyContentsMap[ content.id ] = _.clone( content ); - }); - - self.options = _.defaults( _.omit( options, 'historyContents', 'jobs' ), { + var self = this; + self.options = _.defaults( options, { excludeSetMetadata : false }); self.filters = self._initFilters(); -//TODO: O( 3N ) - self.preprocessJobs( _.clone( jobsJSON ) ); - self.createGraph(); - + _super.prototype.init.call( self, options ); return self; }; @@ -92,127 +90,233 @@ return filters; }; -/** sort the jobs and cache any that pass all filters into _idMap by job.id */ +/** */ +JobDAG.prototype.read = function _read( data ){ + var self = this; + if( _.has( data, 'historyContents' ) && _.has( data, 'jobs' ) && _.has( data, 'tools' ) ){ + // a job dag is composed of these three elements: + // clone the 3 data sources into the DAG, processing the jobs finally using the history and tools + self.preprocessHistoryContents( data.historyContents || [] ) + .preprocessTools( data.tools || {} ) + .preprocessJobs( data.jobs || [] ); + + // filter jobs and create the vertices and edges of the job DAG + self.createGraph( self._filterJobs() ); + return self; + } + return _super.prototype.read.call( this, data ); +}; + +/** */ +JobDAG.prototype.preprocessHistoryContents = function _preprocessHistoryContents( historyContents ){ + this.info( 'processing history' ); + var self = this; + self._historyContentsMap = {}; + + historyContents.forEach( function( content, i ){ + self._historyContentsMap[ content.id ] = _.clone( content ); + }); + return self; +}; + +/** */ +JobDAG.prototype.preprocessTools = function _preprocessTools( tools ){ + this.info( 'processing tools' ); + var self = this; + self._toolMap = {}; + + _.each( tools, function( tool, id ){ + self._toolMap[ id ] = _.clone( tool ); + }); + return self; +}; + +/** sort the cloned jobs, decorate with tool and history contents info, and store in prop array */ JobDAG.prototype.preprocessJobs = function _preprocessJobs( jobs ){ this.info( 'processing jobs' ); + var self = this; + self._outputIdToJobMap = {}; - var self = this; -//TODO:? sorting neccessary? - self.sort( jobs ).forEach( function( job, i ){ - var jobData = self.preprocessJob( job, i ); - if( jobData ){ - self._idMap[ job.id ] = jobData; - } + self._jobsData = self.sort( jobs ).map( function( job ){ + return self.preprocessJob( _.clone( job ) ); }); +//console.debug( JSON.stringify( self._jobsData, null, ' ' ) ); +//console.debug( JSON.stringify( self._outputIdToJobMap, null, ' ' ) ); return self; }; /** sort the jobs based on update time */ JobDAG.prototype.sort = function _sort( jobs ){ function cmpCreate( a, b ){ - if( a.update_time > b.update_time ){ return 1; } - if( a.update_time < b.update_time ){ return -1; } + if( a.create_time > b.create_time ){ return 1; } + if( a.create_time < b.create_time ){ return -1; } return 0; } return jobs.sort( cmpCreate ); }; -/** proces input/output, filter based on job data returning data if passing, null if not */ +/** decorate with input/output datasets and tool */ JobDAG.prototype.preprocessJob = function _preprocessJob( job, index ){ //this.info( 'preprocessJob', job, index ); var self = this, - jobData = { index: index, job: job }; + jobData = { job: job }; - jobData.inputs = self.datasetMapToIdArray( job.inputs, function( dataset, nameInJob ){ - //TODO:? store output name in self._datasets[ output.id ] from creating job? - }); - if( jobData.inputs.length === 0 ){ + jobData.inputs = self._processInputs( job ); + if( _.size( jobData.inputs ) === 0 ){ self.noInputJobs.push( job.id ); } - jobData.outputs = self.datasetMapToIdArray( job.outputs, function( dataset, nameInJob ){ - - }); - if( jobData.outputs.length === 0 ){ + jobData.outputs = self._processOutputs( job ); + if( _.size( jobData.outputs ) === 0 ){ self.noOutputJobs.push( job.id ); } - //self.debug( JSON.stringify( jobData, null, ' ' ) ); - // apply filters after processing job allowing access to the additional data above in the filters - for( var i=0; i<self.filters.length; i++ ){ - if( !self.filters[i].call( self, jobData ) ){ - self.debug( 'job', job.id, ' has been filtered out by function:\n', self.filters[i] ); - return null; - } - } + jobData.tool = self._toolMap[ job.tool_id ]; - //self.info( 'preprocessJob returning', jobData ); + //self.info( '\t jobData:', jobData ); return jobData; }; -/** make verbose input/output lists more concise, sanity checking along the way - * processFn is called on each input/output and passed the dataset obj (id,src) and the input/output name. +/** */ -JobDAG.prototype.datasetMapToIdArray = function _datasetMapToIdArray( datasetMap, processFn ){ +JobDAG.prototype._processInputs = function __processInputs( job ){ + var self = this, + inputs = job.inputs, + inputMap = {}; + _.each( inputs, function( input, nameInJob ){ + input = _.clone( self._validateInputOutput( input ) ); + input.name = nameInJob; + // since this is a DAG and we're processing in order of create time, + // the inputs for this job will already be listed in _outputIdToJobMap + // TODO: we can possibly exploit this + //console.debug( 'input in _outputIdToJobMap', self._outputIdToJobMap[ input.id ] ); + input.content = self._historyContentsMap[ input.id ]; + inputMap[ input.id ] = input; + }); + return inputMap; +}; + +/** + */ +JobDAG.prototype._validateInputOutput = function __validateInputOutput( inputOutput ){ + if( !inputOutput.id ){ + throw new Error( 'No id on job input/output: ', JSON.stringify( inputOutput ) ); + } + if( !inputOutput.src || inputOutput.src !== 'hda' ){ + throw new Error( 'Bad src on job input/output: ', JSON.stringify( inputOutput ) ); + } + return inputOutput; +}; + +/** + */ +JobDAG.prototype._processOutputs = function __processOutputs( job ){ + var self = this, + outputs = job.outputs, + outputMap = {}; + _.each( outputs, function( output, nameInJob ){ + output = _.clone( self._validateInputOutput( output ) ); + output.name = nameInJob; + // add dataset content to jobData + output.content = self._historyContentsMap[ output.id ]; + outputMap[ output.id ] = output; + + self._outputIdToJobMap[ output.id ] = job.id; + }); + return outputMap; +}; + +/** */ +JobDAG.prototype._filterJobs = function __filterJobs(){ var self = this; - return _.map( datasetMap, function( dataset, nameInJob ){ - if( !dataset.id ){ - throw new Error( 'No id on datasetMap: ', JSON.stringify( dataset ) ); + return self._jobsData.filter( function( j, i ){ return self._filterJob( j, i ); }); +}; + +/** + */ +JobDAG.prototype._filterJob = function _filterJob( jobData, index ){ + // apply filters after processing job allowing access to the additional data above inside the filters + var self = this; + for( var i=0; i<self.filters.length; i++ ){ + if( !self.filters[i].call( self, jobData ) ){ + self.debug( '\t job', jobData.job.id, ' has been filtered out by function:\n', self.filters[i] ); + return false; } - if( !dataset.src || dataset.src !== 'hda' ){ - throw new Error( 'Bad src on datasetMap: ', JSON.stringify( dataset ) ); - } - processFn.call( self, dataset, nameInJob ); - return dataset.id; - }); + } + return true; }; /** Walk all the jobs (vertices), attempting to find connections * between datasets used as both inputs and outputs (edges) */ -JobDAG.prototype.createGraph = function _createGraph(){ +JobDAG.prototype.createGraph = function _createGraph( jobsData ){ var self = this; self.debug( 'connections:' ); + //console.debug( jobsData ); - _.each( self._idMap, function( sourceJobData, sourceJobId ){ - self.debug( '\t', sourceJobId, sourceJobData ); - self.createVertex( sourceJobId, sourceJobData ); - - sourceJobData.usedIn = []; - _.each( sourceJobData.outputs, function( outputDatasetId ){ -//TODO: O(n^2) - _.each( self._idMap, function( targetJobData, targetJobId ){ - if( targetJobData === sourceJobData ){ return; } - - if( targetJobData.inputs.indexOf( outputDatasetId ) !== -1 ){ - self.info( '\t\t\t found connection: ', sourceJobId, targetJobId ); - sourceJobData.usedIn.push({ job: targetJobId, output: outputDatasetId }); - - self.createVertex( targetJobId, targetJobData ); - - self.createEdge( sourceJobId, targetJobId, self.directed, { - dataset : outputDatasetId - }); - } + _.each( jobsData, function( jobData ){ + var id = jobData.job.id; + self.debug( '\t', id, jobData ); + self.createVertex( id, jobData ); + }); + _.each( jobsData, function( jobData ){ + var targetId = jobData.job.id; + _.each( jobData.inputs, function( input, inputId ){ + //console.debug( '\t\t target input:', inputId, input ); + var sourceId = self._outputIdToJobMap[ inputId ]; + //console.debug( '\t\t source job id:', sourceId ); + if( !sourceId ){ + var joblessVertex = self.createJobLessVertex( inputId ); + sourceId = joblessVertex.name; + } +//TODO:?? no checking here whether sourceId is actually in the vertex map + //console.debug( '\t\t creating edge, source:', sourceId, self.vertices[ sourceId ] ); + //console.debug( '\t\t creating edge, target:', targetId, self.vertices[ targetId ] ); + self.createEdge( sourceId, targetId, self.directed, { + dataset : inputId }); }); }); - self.debug( 'job data: ', JSON.stringify( self._idMap, null, ' ' ) ); + //console.debug( self.toVerticesAndEdges().edges ); + + self.debug( 'final graph: ', JSON.stringify( self.toVerticesAndEdges(), null, ' ' ) ); return self; }; +/** Return a 'mangled' version of history contents id to prevent contents <-> job id collision */ +JobDAG.prototype.createJobLessVertex = function _createJobLessVertex( contentId ){ + // currently, copied contents are the only history contents without jobs (that I know of) + //note: following needed to prevent id collision btwn content and jobs in vertex map + var JOBLESS_ID_MANGLER = 'copy-', + mangledId = JOBLESS_ID_MANGLER + contentId; + return this.createVertex( mangledId, this._historyContentsMap[ contentId ] ); +}; + /** Override to re-sort (ugh) jobs in each component by update time */ -Graph.prototype.weakComponentGraphArray = function(){ +JobDAG.prototype.weakComponentGraphArray = function(){ + var dag = this; return this.weakComponents().map( function( component ){ //TODO: this seems to belong above (in sort) - why isn't it preserved? + // note: using create_time (as opposed to update_time) + // since update_time for jobless/copied datasets is changes more often component.vertices.sort( function cmpCreate( a, b ){ - if( a.data.job.update_time > b.data.job.update_time ){ return 1; } - if( a.data.job.update_time < b.data.job.update_time ){ return -1; } + var aCreateTime = a.data.job? a.data.job.create_time : a.data.create_time, + bCreateTime = b.data.job? b.data.job.create_time : b.data.create_time; + if( aCreateTime > bCreateTime ){ return 1; } + if( aCreateTime < bCreateTime ){ return -1; } return 0; }); - return new Graph( this.directed, component ); + return new Graph( dag.directed, component ); }); }; +JobDAG.prototype._jobsDataMap = function(){ + var jobsDataMap = {}; + this._jobsData.forEach( function( jobData ){ + jobsDataMap[ jobData.job.id ] = jobData; + }); + return jobsDataMap; +}; + // ============================================================================ return JobDAG; diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff client/galaxy/scripts/mvc/job/job-li.js --- a/client/galaxy/scripts/mvc/job/job-li.js +++ b/client/galaxy/scripts/mvc/job/job-li.js @@ -14,7 +14,6 @@ /** logger used to record this.log messages, commonly set to console */ //logger : console, - className : _super.prototype.className + " job", id : function(){ return [ 'job', this.model.get( 'id' ) ].join( '-' ); @@ -28,6 +27,9 @@ this.log( this + '.initialize:', attributes ); _super.prototype.initialize.call( this, attributes ); + this.tool = attributes.tool || {}; + this.jobData = attributes.jobData || {}; + /** where should pages from links be displayed? (default to new tab/window) */ this.linkTarget = attributes.linkTarget || '_blank'; }, @@ -50,6 +52,61 @@ }); }, + // ........................................................................ template helpers + // all of these are ADAPTERs - in other words, it might be better if the API returned the final form + // or something similar in order to remove some of the complexity here + + /** Return tool.inputs that should/can be safely displayed */ + _labelParamMap : function(){ + //ADAPTER + var params = this.model.get( 'params' ), + labelParamMap = {}; + _.each( this.tool.inputs, function( i ){ + //console.debug( i.label, i.model_class ); + if( i.label && i.model_class !== 'DataToolParameter' ){ + labelParamMap[ i.label ] = params[ i.name ]; + } + }); + return labelParamMap; + }, + + _labelInputMap : function(){ + //ADAPTER + var view = this, + labelInputMap = {}; + _.each( this.jobData.inputs, function( input ){ + var toolInput = view._findToolInput( input.name ); + if( toolInput ){ + labelInputMap[ toolInput.label ] = input; + } + }); + return labelInputMap; + }, + + /** Return a tool.inputs object that matches (or partially matches) the given (job input) name */ + _findToolInput : function( name ){ + //ADAPTER + var toolInputs = this.tool.inputs, + exactMatch = _.findWhere( toolInputs, { name : name }); + if( exactMatch ){ return exactMatch; } + return this._findRepeatToolInput( name, toolInputs ); + }, + + /** Return a tool.inputs object that partially matches the given (job input) name (for repeat dataset inputs)*/ + _findRepeatToolInput : function( name, toolInputs ){ + //ADAPTER + toolInputs = toolInputs || this.tool.inputs; + var partialMatch = _.find( toolInputs, function( i ){ + return name.indexOf( i.name ) === 0; + }); + if( !partialMatch ){ return undefined; } + + var subMatch = _.find( partialMatch.inputs, function( i ){ + return name.indexOf( i.name ) !== -1; + }); + return subMatch; + }, + // ........................................................................ misc /** String representation */ toString : function(){ @@ -86,38 +143,58 @@ '<div class="title-bar clear" tabindex="0">', //'<span class="state-icon"></span>', '<div class="title">', - '<span class="name"><%- job.tool_id %></span>', + '<span class="name"><%- view.tool.name %></span>', '</div>', - '<div class="subtitle"></div>', + '<div class="subtitle">', + '<span class="description"><%- view.tool.description %></span', + '<span class="create-time">', + ' ', _l( 'Created' ), ': <%= new Date( job.create_time ).toString() %>, ', + '</span', + '</div>', '</div>' ], 'job' ); - //var subtitleTemplate = BASE_MVC.wrapTemplate([ - // // override this - // '<div class="subtitle">', - // _l( 'Created' ), ': <%= new Date( job.create_time ).toString() %>, ', - // _l( 'Updated' ), ': <%= new Date( job.update_time ).toString() %>', - // '</div>' - //], 'job' ); - // - //var detailsTemplate = BASE_MVC.wrapTemplate([ - // '<div class="details">', - // '<div class="params">', - // '<% _.each( job.params, function( param, paramName ){ %>', - // '<div class="param">', - // '<label class="prompt"><%= paramName %></label>', - // '<span class="value"><%= param %></span>', - // '</div>', - // '<% }) %>', - // '</div>', - // '</div>' - //], 'job' ); + var subtitleTemplate = BASE_MVC.wrapTemplate([ + '<div class="subtitle">', + '<span class="description"><%- view.tool.description %></span', + //'<span class="create-time">', + // ' ', _l( 'Created' ), ': <%= new Date( job.create_time ).toString() %>, ', + //'</span', + //'<span class="version">', + // ' (', _l( 'version' ), ': <%= view.tool.version %>)', + //'</span', + '</div>' + ], 'job' ); + + var detailsTemplate = BASE_MVC.wrapTemplate([ + '<div class="details">', + //'<div class="version">', + // '<label class="prompt">', _l( 'Version' ), '</label>', + // '<span class="value"><%= view.tool.version %></span>', + //'</div>', + '<div class="params">', + '<% _.each( view._labelInputMap(), function( input, label ){ %>', + '<div class="input" data-input-name="<%= input.name %>" data-input-id="<%= input.id %>">', + '<label class="prompt"><%= label %></label>', +//TODO: input dataset name + '<span class="value"><%= input.content.name %></span>', + '</div>', + '<% }) %>', + '<% _.each( view._labelParamMap(), function( param, label ){ %>', + '<div class="param" data-input-name="<%= param.name %>">', + '<label class="prompt"><%= label %></label>', + '<span class="value"><%= param %></span>', + '</div>', + '<% }) %>', + '</div>', + '</div>' + ], 'job' ); return _.extend( {}, _super.prototype.templates, { - //el : elTemplate, + //el : elTemplate, titleBar : titleBarTemplate, - //subtitle : subtitleTemplate, - //details : detailsTemplate + subtitle : subtitleTemplate, + details : detailsTemplate }); }()); diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff client/galaxy/scripts/mvc/job/job-model.js --- a/client/galaxy/scripts/mvc/job/job-model.js +++ b/client/galaxy/scripts/mvc/job/job-model.js @@ -19,7 +19,7 @@ defaults : { model_class : 'Job', - tool : null, + tool_id : null, exit_code : null, inputs : {}, @@ -31,9 +31,27 @@ state : STATES.NEW }, + /** override to parse params on incomming */ + parse : function( response, options ){ + response.params = this.parseParams( response.params ); + return response; + }, + + /** override to treat param values as json */ + parseParams : function( params ){ + var newParams = {}; + _.each( params, function( value, key ){ + newParams[ key ] = JSON.parse( value ); + }); + return newParams; + }, + /** instance vars and listeners */ initialize : function( attributes, options ){ this.debug( this + '(Job).initialize', attributes, options ); + + this.set( 'params', this.parseParams( this.get( 'params' ) ), { silent: true }); + this.outputCollection = attributes.outputCollection || new HISTORY_CONTENTS.HistoryContents([]); this._setUpListeners(); }, @@ -62,7 +80,7 @@ /** Does this model already contain detailed data (as opposed to just summary level data)? */ hasDetails : function(){ //?? this may not be reliable - return _.isEmpty( this.get( 'outputs' ) ); + return !_.isEmpty( this.get( 'outputs' ) ); }, // ........................................................................ ajax @@ -73,9 +91,9 @@ // ........................................................................ searching // see base-mvc, SearchableModelMixin /** what attributes of an Job will be used in a text search */ - searchAttributes : [ - 'tool' - ], + //searchAttributes : [ + // 'tool' + //], // ........................................................................ misc /** String representation */ diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff client/galaxy/scripts/mvc/list/list-panel.js --- a/client/galaxy/scripts/mvc/list/list-panel.js +++ b/client/galaxy/scripts/mvc/list/list-panel.js @@ -514,6 +514,7 @@ /** get views based on model */ viewFromModel : function( model ){ + if( !model ){ return undefined; } return this.viewFromModelId( model.id ); }, diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff client/galaxy/scripts/utils/graph.js --- a/client/galaxy/scripts/utils/graph.js +++ b/client/galaxy/scripts/utils/graph.js @@ -596,8 +596,9 @@ /** Return an array of graphs of the weakly connected components in this graph */ Graph.prototype.weakComponentGraphArray = function(){ //note: although this can often look like the original graph - edges can be lost + var graph = this; return this.weakComponents().map( function( component ){ - return new Graph( this.directed, component ); + return new Graph( graph.directed, component ); }); }; diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2019,6 +2019,7 @@ state = hda.state, history_content_type=hda.history_content_type, file_size = int( hda.get_size() ), + create_time = hda.create_time.isoformat(), update_time = hda.update_time.isoformat(), data_type = hda.datatype.__class__.__module__ + '.' + hda.datatype.__class__.__name__, genome_build = hda.dbkey, @@ -2034,6 +2035,10 @@ tags_str_list.append( tag_str ) rval[ 'tags' ] = tags_str_list + #if getattr( hda, 'hidden_beneath_collection_instance', False ): + # collection_id = hda.hidden_beneath_collection_instance.id + # rval['collection_id'] = collection_id + if hda.copied_from_library_dataset_dataset_association is not None: rval['copied_from_ldda_id'] = hda.copied_from_library_dataset_dataset_association.id diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -1,9 +1,11 @@ import logging from cgi import escape +import urllib import galaxy.util from galaxy import model from galaxy import web +from galaxy import exceptions from galaxy import managers from galaxy.datatypes.data import nice_size from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings @@ -529,7 +531,7 @@ return trans.fill_template( "history/display_structured.mako", items=items, history=history ) @web.expose - def structure( self, trans, id=None ): + def structure( self, trans, id=None, **kwargs ): """ """ if not id: @@ -543,11 +545,19 @@ jobs = ( trans.sa_session.query( trans.app.model.Job ) .filter( trans.app.model.Job.user == trans.user ) .filter( trans.app.model.Job.history_id == trans.security.decode_id( id ) ) ).all() - jobs = map( lambda j: self.encode_all_ids( trans, j.to_dict( 'element' ), True ), jobs ) + tools = {} + for tool_id in set( map( lambda j: j[ 'tool_id' ], jobs ) ): + unquoted_id = urllib.unquote_plus( tool_id ) + tool = self.app.toolbox.get_tool( unquoted_id ) + if not tool: + raise exceptions.ObjectNotFound( "Could not find tool with id '%s'" % tool_id ) + #TODO: some fallback for tool information + tools[ tool_id ] = tool.to_dict( trans, io_details=True, link_details=True ) + return trans.fill_template( "history/structure.mako", historyId=history_dictionary[ 'id' ], - history=history_dictionary, hdas=hda_dictionaries, jobs=jobs ) + history=history_dictionary, hdas=hda_dictionaries, jobs=jobs, tools=tools, **kwargs ) @web.expose def view( self, trans, id=None, show_deleted=False, show_hidden=False, use_panels=True ): diff -r 6fd14d9f8b099e926a2b8fa462938d8955c1713c -r 5f0d3cbd97d9c1472bdb8249f3df72d6683caaff static/scripts/mvc/history/history-panel-edit-current.js --- a/static/scripts/mvc/history/history-panel-edit-current.js +++ b/static/scripts/mvc/history/history-panel-edit-current.js @@ -164,12 +164,13 @@ _setUpCollectionListeners : function(){ _super.prototype._setUpCollectionListeners.call( this ); + //TODO:?? may not be needed? see history-panel-edit, 369 // if a hidden item is created (gen. by a workflow), moves thru the updater to the ready state, // then: remove it from the collection if the panel is set to NOT show hidden datasets this.collection.on( 'state:ready', function( model, newState, oldState ){ if( ( !model.get( 'visible' ) ) && ( !this.storage.get( 'show_hidden' ) ) ){ - this.removeItemView( this.viewFromModel( model ) ); + this.removeItemView( model ); } }, this ); }, This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.