1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/4fbd05095ca7/ changeset: 4fbd05095ca7 user: jgoecks date: 2012-06-12 23:37:58 summary: Infrastructure for running tools: (a) move rerunning tools from tracks controller into tools (API) controller; (b) rerunning now supports multiple regions. affected #: 11 files diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2617,7 +2617,7 @@ elif isinstance( input, SelectToolParameter ): param_dict.update( { 'type' : 'select', 'html' : urllib.quote( input.get_html( trans ) ), - 'options': input.static_options + 'options': input.static_options } ) elif isinstance( input, Conditional ): # TODO. @@ -2626,7 +2626,8 @@ param_dict.update( { 'type' : 'number', 'init_value' : input.value, 'html' : urllib.quote( input.get_html( trans ) ), 'min': input.min, - 'max': input.max + 'max': input.max, + 'value': input.value } ) else: param_dict.update( { 'type' : '??', 'init_value' : input.value, \ diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/visualization/genomes.py --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -23,6 +23,26 @@ return dbkey.split( ':' ) else: return None, dbkey + +class GenomeRegion( object ): + """ + A genomic region on an individual chromosome. + """ + + def __init__( self, chrom=None, start=None, end=None ): + self.chrom = chrom + self.start = int( start ) + self.end = int( end ) + + def __str__( self ): + return self.chrom + ":" + str( self.start ) + "-" + str( self.end ) + + @staticmethod + def from_dict( obj_dict ): + return GenomeRegion( chrom=obj_dict[ 'chrom' ], + start=obj_dict[ 'start' ], + end=obj_dict[ 'end' ] ) + class Genome( object ): """ diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/visualization/tracks/data_providers.py --- a/lib/galaxy/visualization/tracks/data_providers.py +++ b/lib/galaxy/visualization/tracks/data_providers.py @@ -63,7 +63,7 @@ self.original_dataset = original_dataset self.dependencies = dependencies - def write_data_to_file( self, chrom, start, end, filename ): + def write_data_to_file( self, regions, filename ): """ Write data in region defined by chrom, start, and end to a file. """ @@ -257,11 +257,18 @@ return tabix.fetch(reference=chrom, start=start, end=end) - def write_data_to_file( self, chrom, start, end, filename ): - iterator = self.get_iterator( chrom, start, end ) + def write_data_to_file( self, regions, filename ): out = open( filename, "w" ) - for line in iterator: - out.write( "%s\n" % line ) + + for region in regions: + # Write data in region. + chrom = region.chrom + start = region.start + end = region.end + iterator = self.get_iterator( chrom, start, end ) + for line in iterator: + out.write( "%s\n" % line ) + out.close() # @@ -332,7 +339,7 @@ return { 'data': rval, 'message': message } - def write_data_to_file( self, chrom, start, end, filename ): + def write_data_to_file( self, regions, filename ): raise Exception( "Unimplemented Function" ) class IntervalTabixDataProvider( TabixDataProvider, IntervalDataProvider ): @@ -420,11 +427,18 @@ return { 'data': rval, 'message': message } - def write_data_to_file( self, chrom, start, end, filename ): - iterator = self.get_iterator( chrom, start, end ) + def write_data_to_file( self, regions, filename ): out = open( filename, "w" ) - for line in iterator: - out.write( "%s\n" % line ) + + for region in regions: + # Write data in region. + chrom = region.chrom + start = region.start + end = region.end + iterator = self.get_iterator( chrom, start, end ) + for line in iterator: + out.write( "%s\n" % line ) + out.close() class BedTabixDataProvider( TabixDataProvider, BedDataProvider ): @@ -545,11 +559,17 @@ return { 'data': rval, 'message': message } - def write_data_to_file( self, chrom, start, end, filename ): - iterator = self.get_iterator( chrom, start, end ) + def write_data_to_file( self, regions, filename ): out = open( filename, "w" ) - for line in iterator: - out.write( "%s\n" % line ) + + for region in regions: + # Write data in region. + chrom = region.chrom + start = region.start + end = region.end + iterator = self.get_iterator( chrom, start, end ) + for line in iterator: + out.write( "%s\n" % line ) out.close() class VcfTabixDataProvider( TabixDataProvider, VcfDataProvider ): @@ -669,35 +689,42 @@ return filters - def write_data_to_file( self, chrom, start, end, filename ): + def write_data_to_file( self, regions, filename ): """ - Write reads in [chrom:start-end] to file. + Write reads in regions to file. """ # Open current BAM file using index. - start, end = int(start), int(end) bamfile = csamtools.Samfile( filename=self.original_dataset.file_name, mode='rb', \ index_filename=self.converted_dataset.file_name ) - try: - data = bamfile.fetch(start=start, end=end, reference=chrom) - except ValueError, e: - # Some BAM files do not prefix chromosome names with chr, try without - if chrom.startswith( 'chr' ): - try: - data = bamfile.fetch( start=start, end=end, reference=chrom[3:] ) - except ValueError: - return None - else: - return None - - # Write new BAM file. + # TODO: write headers as well? new_bamfile = csamtools.Samfile( template=bamfile, filename=filename, mode='wb' ) - for i, read in enumerate( data ): - new_bamfile.write( read ) - new_bamfile.close() + + for region in regions: + # Write data from region. + chrom = region.chrom + start = region.start + end = region.end + + try: + data = bamfile.fetch(start=start, end=end, reference=chrom) + except ValueError, e: + # Some BAM files do not prefix chromosome names with chr, try without + if chrom.startswith( 'chr' ): + try: + data = bamfile.fetch( start=start, end=end, reference=chrom[3:] ) + except ValueError: + return None + else: + return None + + # Write reads in region. + for i, read in enumerate( data ): + new_bamfile.write( read ) # Cleanup. + new_bamfile.close() bamfile.close() def get_iterator( self, chrom, start, end ): @@ -952,17 +979,24 @@ """ col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } - def write_data_to_file( self, chrom, start, end, filename ): + def write_data_to_file( self, regions, filename ): source = open( self.original_dataset.file_name ) index = Indexes( self.converted_dataset.file_name ) out = open( filename, 'w' ) - for start, end, offset in index.find(chrom, start, end): - source.seek( offset ) + + for region in regions: + # Write data from region. + chrom = region.chrom + start = region.start + end = region.end + for start, end, offset in index.find(chrom, start, end): + source.seek( offset ) - reader = GFFReaderWrapper( source, fix_strand=True ) - feature = reader.next() - for interval in feature.intervals: - out.write( '\t'.join( interval.fields ) + '\n' ) + reader = GFFReaderWrapper( source, fix_strand=True ) + feature = reader.next() + for interval in feature.intervals: + out.write( '\t'.join( interval.fields ) + '\n' ) + out.close() def get_iterator( self, chrom, start, end ): @@ -1183,13 +1217,6 @@ rval.append( payload ) return { 'data': rval, 'message': message } - - def write_data_to_file( self, chrom, start, end, filename ): - iterator = self.get_iterator( chrom, start, end ) - out = open( filename, "w" ) - for line in iterator: - out.write( "%s\n" % line ) - out.close() class ENCODEPeakTabixDataProvider( TabixDataProvider, ENCODEPeakDataProvider ): """ diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/web/api/datasets.py --- a/lib/galaxy/web/api/datasets.py +++ b/lib/galaxy/web/api/datasets.py @@ -10,7 +10,7 @@ log = logging.getLogger( __name__ ) -class DatasetsController( BaseAPIController, UsesHistoryMixinDatasetAssociationMixin ): +class DatasetsController( BaseAPIController, UsesHistoryDatasetAssociationMixin ): @web.expose_api def index( self, trans, hda_id, **kwd ): diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/web/api/history_contents.py --- a/lib/galaxy/web/api/history_contents.py +++ b/lib/galaxy/web/api/history_contents.py @@ -12,7 +12,7 @@ log = logging.getLogger( __name__ ) -class HistoryContentsController( BaseAPIController, UsesHistoryMixinDatasetAssociationMixin, UsesHistoryMixin, UsesLibraryMixin, UsesLibraryMixinItems ): +class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesHistoryMixin, UsesLibraryMixin, UsesLibraryMixinItems ): @web.expose_api def index( self, trans, history_id, **kwd ): diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/web/api/tools.py --- a/lib/galaxy/web/api/tools.py +++ b/lib/galaxy/web/api/tools.py @@ -1,12 +1,12 @@ from galaxy import config, tools, web, util -from galaxy.web.base.controller import BaseController, BaseAPIController +from galaxy.web.base.controller import BaseController, BaseAPIController, UsesHistoryDatasetAssociationMixin, messages, get_highest_priority_msg from galaxy.util.bunch import Bunch +from galaxy.visualization.tracks.visual_analytics import get_dataset_job +from galaxy.visualization.genomes import GenomeRegion +from galaxy.util.json import to_json_string, from_json_string +from galaxy.visualization.tracks.data_providers import * -messages = Bunch( - NO_TOOL = "no tool" -) - -class ToolsController( BaseAPIController ): +class ToolsController( BaseAPIController, UsesHistoryDatasetAssociationMixin ): """ RESTful controller for interactions with tools. """ @@ -29,7 +29,7 @@ # Create return value. return self.app.toolbox.to_dict( trans, in_panel=in_panel, trackster=trackster ) - @web.json + @web.expose_api def show( self, trans, id, **kwd ): """ GET /api/tools/{tool_id} @@ -41,16 +41,18 @@ def create( self, trans, payload, **kwd ): """ POST /api/tools - Executes tool using specified inputs, creating new history-dataset - associations, which are returned. + Executes tool using specified inputs and returns tool's outputs. """ - # TODO: set target history? + # HACK: for now, if action is rerun, rerun tool. + action = payload.get( 'action', None ) + if action == 'rerun': + return self._rerun_tool( trans, payload, **kwd ) # -- Execute tool. -- # Get tool. - tool_id = payload[ 'id' ] + tool_id = payload[ 'tool_id' ] tool = trans.app.toolbox.get_tool( tool_id ) if not tool: return { "message": { "type": "error", "text" : messages.NO_TOOL } } @@ -72,4 +74,287 @@ for output in output_datasets: outputs.append( output.get_api_value() ) return rval - \ No newline at end of file + + # + # -- Helper methods -- + # + + def _run_tool( self, trans, tool_id, target_dataset_id, **kwargs ): + """ + Run a tool. This method serves as a general purpose way to run tools asynchronously. + """ + + # + # Set target history (the history that tool will use for outputs) using + # target dataset. If user owns dataset, put new data in original + # dataset's history; if user does not own dataset (and hence is accessing + # dataset via sharing), put new data in user's current history. + # + target_dataset = self.get_dataset( trans, target_dataset_id, check_ownership=False, check_accessible=True ) + if target_dataset.history.user == trans.user: + target_history = target_dataset.history + else: + target_history = trans.get_history( create=True ) + + # HACK: tools require unencoded parameters but kwargs are typically + # encoded, so try decoding all parameter values. + for key, value in kwargs.items(): + try: + value = trans.security.decode_id( value ) + kwargs[ key ] = value + except: + pass + + # + # Execute tool. + # + tool = trans.app.toolbox.get_tool( tool_id ) + if not tool: + return messages.NO_TOOL + + # HACK: add run button so that tool.handle_input will run tool. + kwargs['runtool_btn'] = 'Execute' + params = util.Params( kwargs, sanitize = False ) + template, vars = tool.handle_input( trans, params.__dict__, history=target_history ) + + # TODO: check for errors and ensure that output dataset is available. + output_datasets = vars[ 'out_data' ].values() + return self.add_track_async( trans, output_datasets[0].id ) + + + def _rerun_tool( self, trans, payload, **kwargs ): + """ + Rerun a tool to produce a new output dataset that corresponds to a + dataset that a user is currently viewing. + """ + + # + # TODO: refactor to use same code as run_tool. + # + + # Run tool on region if region is specificied. + run_on_regions = False + regions = from_json_string( payload.get( 'regions', None ) ) + print regions, payload + if regions: + if isinstance( regions, dict ): + # Regions is a single region. + regions = [ GenomeRegion.from_dict( regions ) ] + elif isinstance( regions, list ): + # There is a list of regions. + regions = [ GenomeRegion.from_dict( r ) for r in regions ] + run_on_regions = True + + # Dataset check. + original_dataset = self.get_dataset( trans, payload[ 'target_dataset_id' ], check_ownership=False, check_accessible=True ) + msg = self.check_dataset_state( trans, original_dataset ) + if msg: + return to_json_string( msg ) + + # + # Set tool parameters--except non-hidden dataset parameters--using combination of + # job's previous parameters and incoming parameters. Incoming parameters + # have priority. + # + original_job = get_dataset_job( original_dataset ) + tool = trans.app.toolbox.get_tool( original_job.tool_id ) + if not tool: + return messages.NO_TOOL + tool_params = dict( [ ( p.name, p.value ) for p in original_job.parameters ] ) + # TODO: need to handle updates to conditional parameters; conditional + # params are stored in dicts (and dicts within dicts). + tool_params.update( dict( [ ( key, value ) for key, value in kwargs.items() if key in tool.inputs ] ) ) + tool_params = tool.params_from_strings( tool_params, self.app ) + + # + # If running tool on region, convert input datasets (create indices) so + # that can regions of data can be quickly extracted. + # + messages_list = [] + if run_on_regions: + for jida in original_job.input_datasets: + input_dataset = jida.dataset + if get_data_provider( original_dataset=input_dataset ): + # Can index dataset. + track_type, data_sources = input_dataset.datatype.get_track_type() + # Convert to datasource that provides 'data' because we need to + # extract the original data. + data_source = data_sources[ 'data' ] + msg = self.convert_dataset( trans, input_dataset, data_source ) + if msg is not None: + messages_list.append( msg ) + + # Return any messages generated during conversions. + return_message = get_highest_priority_msg( messages_list ) + if return_message: + return to_json_string( return_message ) + + # + # Set target history (the history that tool will use for inputs/outputs). + # If user owns dataset, put new data in original dataset's history; if + # user does not own dataset (and hence is accessing dataset via sharing), + # put new data in user's current history. + # + if original_dataset.history.user == trans.user: + target_history = original_dataset.history + else: + target_history = trans.get_history( create=True ) + hda_permissions = trans.app.security_agent.history_get_default_permissions( target_history ) + + def set_param_value( param_dict, param_name, param_value ): + """ + Set new parameter value in a tool's parameter dictionary. + """ + + # Recursive function to set param value. + def set_value( param_dict, group_name, group_index, param_name, param_value ): + if group_name in param_dict: + param_dict[ group_name ][ group_index ][ param_name ] = param_value + return True + elif param_name in param_dict: + param_dict[ param_name ] = param_value + return True + else: + # Recursive search. + return_val = False + for name, value in param_dict.items(): + if isinstance( value, dict ): + return_val = set_value( value, group_name, group_index, param_name, param_value) + if return_val: + return return_val + return False + + # Parse parameter name if necessary. + if param_name.find( "|" ) == -1: + # Non-grouping parameter. + group_name = group_index = None + else: + # Grouping parameter. + group, param_name = param_name.split( "|" ) + index = group.rfind( "_" ) + group_name = group[ :index ] + group_index = int( group[ index + 1: ] ) + + return set_value( param_dict, group_name, group_index, param_name, param_value ) + + # Set parameters based tool's trackster config. + params_set = {} + for action in tool.trackster_conf.actions: + success = False + for joda in original_job.output_datasets: + if joda.name == action.output_name: + set_param_value( tool_params, action.name, joda.dataset ) + params_set[ action.name ] = True + success = True + break + + if not success: + return messages.ERROR + + # + # Set input datasets for tool. If running on regions, extract and use subset + # when possible. + # + regions_str = ",".join( [ str( r ) for r in regions ] ) + for jida in original_job.input_datasets: + # If param set previously by config actions, do nothing. + if jida.name in params_set: + continue + + input_dataset = jida.dataset + if input_dataset is None: #optional dataset and dataset wasn't selected + tool_params[ jida.name ] = None + elif run_on_regions and hasattr( input_dataset.datatype, 'get_track_type' ): + # Dataset is indexed and hence a subset can be extracted and used + # as input. + + # Look for subset. + subset_dataset_association = trans.sa_session.query( trans.app.model.HistoryDatasetAssociationSubset ) \ + .filter_by( hda=input_dataset, location=regions_str ) \ + .first() + if subset_dataset_association: + # Data subset exists. + subset_dataset = subset_dataset_association.subset + else: + # Need to create subset. + track_type, data_sources = input_dataset.datatype.get_track_type() + data_source = data_sources[ 'data' ] + converted_dataset = input_dataset.get_converted_dataset( trans, data_source ) + deps = input_dataset.get_converted_dataset_deps( trans, data_source ) + + # Create new HDA for input dataset's subset. + new_dataset = trans.app.model.HistoryDatasetAssociation( extension=input_dataset.ext, \ + dbkey=input_dataset.dbkey, \ + create_dataset=True, \ + sa_session=trans.sa_session, + name="Subset [%s] of data %i" % \ + ( regions_str, input_dataset.hid ), + visible=False ) + target_history.add_dataset( new_dataset ) + trans.sa_session.add( new_dataset ) + trans.app.security_agent.set_all_dataset_permissions( new_dataset.dataset, hda_permissions ) + + # Write subset of data to new dataset + data_provider_class = get_data_provider( original_dataset=input_dataset ) + data_provider = data_provider_class( original_dataset=input_dataset, + converted_dataset=converted_dataset, + dependencies=deps ) + trans.app.object_store.create( new_dataset.dataset ) + data_provider.write_data_to_file( regions, new_dataset.file_name ) + + # TODO: (a) size not working; (b) need to set peek. + new_dataset.set_size() + new_dataset.info = "Data subset for trackster" + new_dataset.set_dataset_state( trans.app.model.Dataset.states.OK ) + + # Set metadata. + # TODO: set meta internally if dataset is small enough? + if trans.app.config.set_metadata_externally: + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, + trans, incoming = { 'input1':new_dataset }, + overwrite=False, job_params={ "source" : "trackster" } ) + else: + message = 'Attributes updated' + new_dataset.set_meta() + new_dataset.datatype.after_setting_metadata( new_dataset ) + + # Add HDA subset association. + subset_association = trans.app.model.HistoryDatasetAssociationSubset( hda=input_dataset, subset=new_dataset, location=regions_str ) + trans.sa_session.add( subset_association ) + + subset_dataset = new_dataset + + trans.sa_session.flush() + + # Add dataset to tool's parameters. + if not set_param_value( tool_params, jida.name, subset_dataset ): + return to_json_string( { "error" : True, "message" : "error setting parameter %s" % jida.name } ) + + # + # Execute tool and handle outputs. + # + try: + subset_job, subset_job_outputs = tool.execute( trans, incoming=tool_params, + history=target_history, + job_params={ "source" : "trackster" } ) + except Exception, e: + # Lots of things can go wrong when trying to execute tool. + return to_json_string( { "error" : True, "message" : e.__class__.__name__ + ": " + str(e) } ) + if run_on_regions: + for output in subset_job_outputs.values(): + output.visible = False + trans.sa_session.flush() + + # + # Return new track that corresponds to the original dataset. + # + output_name = None + for joda in original_job.output_datasets: + if joda.dataset == original_dataset: + output_name = joda.name + break + for joda in subset_job.output_datasets: + if joda.name == output_name: + output_dataset = joda.dataset + + return output_dataset.get_api_value() diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -181,6 +181,37 @@ # -- Mixins for working with Galaxy objects. -- # +# Message strings returned to browser +messages = Bunch( + PENDING = "pending", + NO_DATA = "no data", + NO_CHROMOSOME = "no chromosome", + NO_CONVERTER = "no converter", + NO_TOOL = "no tool", + DATA = "data", + ERROR = "error", + OK = "ok" +) + +def get_highest_priority_msg( message_list ): + """ + Returns highest priority message from a list of messages. + """ + return_message = None + + # For now, priority is: job error (dict), no converter, pending. + for message in message_list: + if message is not None: + if isinstance(message, dict): + return_message = message + break + elif message == messages.NO_CONVERTER: + return_message = message + elif return_message == None and message == messages.PENDING: + return_message = message + return return_message + + class SharableItemSecurityMixin: """ Mixin for handling security for sharable items. """ def security_check( self, trans, item, check_ownership=False, check_accessible=False ): @@ -201,8 +232,9 @@ raise ItemAccessibilityException( "%s is not accessible to the current user" % item.__class__.__name__, type='error' ) return item -class UsesHistoryMixinDatasetAssociationMixin: +class UsesHistoryDatasetAssociationMixin: """ Mixin for controllers that use HistoryDatasetAssociation objects. """ + def get_dataset( self, trans, dataset_id, check_ownership=True, check_accessible=False ): """ Get an HDA object by id. """ # DEPRECATION: We still support unencoded ids for backward compatibility @@ -232,6 +264,7 @@ else: error( "You are not allowed to access this dataset" ) return data + def get_history_dataset_association( self, trans, history, dataset_id, check_ownership=True, check_accessible=False ): """Get a HistoryDatasetAssociation from the database by id, verifying ownership.""" self.security_check( trans, history, check_ownership=check_ownership, check_accessible=check_accessible ) @@ -244,6 +277,7 @@ else: error( "You are not allowed to access this dataset" ) return hda + def get_data( self, dataset, preview=True ): """ Gets a dataset's data. """ # Get data from file, truncating if necessary. @@ -258,6 +292,46 @@ dataset_data = open( dataset.file_name ).read(max_peek_size) truncated = False return truncated, dataset_data + + def check_dataset_state( self, trans, dataset ): + """ + Returns a message if dataset is not ready to be used in visualization. + """ + if not dataset: + return messages.NO_DATA + if dataset.state == trans.app.model.Job.states.ERROR: + return messages.ERROR + if dataset.state != trans.app.model.Job.states.OK: + return messages.PENDING + return None + + def convert_dataset( self, trans, dataset, target_type ): + """ + Converts a dataset to the target_type and returns a message indicating + status of the conversion. None is returned to indicate that dataset + was converted successfully. + """ + + # Get converted dataset; this will start the conversion if necessary. + try: + converted_dataset = dataset.get_converted_dataset( trans, target_type ) + except NoConverterException: + return messages.NO_CONVERTER + except ConverterDependencyException, dep_error: + return { 'kind': messages.ERROR, 'message': dep_error.value } + + # Check dataset state and return any messages. + msg = None + if converted_dataset and converted_dataset.state == trans.app.model.Dataset.states.ERROR: + job_id = trans.sa_session.query( trans.app.model.JobToOutputDatasetAssociation ) \ + .filter_by( dataset_id=converted_dataset.id ).first().job_id + job = trans.sa_session.query( trans.app.model.Job ).get( job_id ) + msg = { 'kind': messages.ERROR, 'message': job.stderr } + elif not converted_dataset or converted_dataset.state != trans.app.model.Dataset.states.OK: + msg = messages.PENDING + + return msg + class UsesLibraryMixin: def get_library( self, trans, id, check_ownership=False, check_accessible=True ): diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py +++ b/lib/galaxy/web/controllers/dataset.py @@ -150,7 +150,7 @@ .filter( model.History.deleted==False ) \ .filter( self.model_class.visible==True ) -class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, UsesHistoryMixinDatasetAssociationMixin, UsesItemRatings ): +class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, UsesItemRatings ): stored_list_grid = HistoryDatasetAssociationListGrid() diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/web/controllers/page.py --- a/lib/galaxy/web/controllers/page.py +++ b/lib/galaxy/web/controllers/page.py @@ -273,7 +273,7 @@ _BaseHTMLProcessor.unknown_endtag( self, tag ) class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHistoryMixin, - UsesStoredWorkflowMixin, UsesHistoryMixinDatasetAssociationMixin, UsesVisualizationMixin, UsesItemRatings ): + UsesStoredWorkflowMixin, UsesHistoryDatasetAssociationMixin, UsesVisualizationMixin, UsesItemRatings ): _page_list = PageListGrid() _all_published_list = PageAllPublishedGrid() diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/web/controllers/tracks.py --- a/lib/galaxy/web/controllers/tracks.py +++ b/lib/galaxy/web/controllers/tracks.py @@ -18,17 +18,6 @@ from galaxy.visualization.genomes import decode_dbkey, Genomes from galaxy.visualization.tracks.visual_analytics import get_tool_def, get_dataset_job -# Message strings returned to browser -messages = Bunch( - PENDING = "pending", - NO_DATA = "no data", - NO_CHROMOSOME = "no chromosome", - NO_CONVERTER = "no converter", - NO_TOOL = "no tool", - DATA = "data", - ERROR = "error", - OK = "ok" -) class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, history ): @@ -163,7 +152,7 @@ def apply_query_filter( self, trans, query, **kwargs ): return query.filter( self.model_class.user_id == trans.user.id ) -class TracksController( BaseUIController, UsesVisualizationMixin, UsesHistoryMixinDatasetAssociationMixin, SharableMixin ): +class TracksController( BaseUIController, UsesVisualizationMixin, UsesHistoryDatasetAssociationMixin, SharableMixin ): """ Controller for track browser interface. Handles building a new browser from datasets in the current history, and display of the resulting browser. @@ -488,281 +477,7 @@ @web.expose def list_tracks( self, trans, **kwargs ): return self.tracks_grid( trans, **kwargs ) - - @web.expose - def run_tool( self, trans, tool_id, target_dataset_id, **kwargs ): - """ - Run a tool. This method serves as a general purpose way to run tools asynchronously. - """ - - # - # Set target history (the history that tool will use for outputs) using - # target dataset. If user owns dataset, put new data in original - # dataset's history; if user does not own dataset (and hence is accessing - # dataset via sharing), put new data in user's current history. - # - target_dataset = self.get_dataset( trans, target_dataset_id, check_ownership=False, check_accessible=True ) - if target_dataset.history.user == trans.user: - target_history = target_dataset.history - else: - target_history = trans.get_history( create=True ) - - # HACK: tools require unencoded parameters but kwargs are typically - # encoded, so try decoding all parameter values. - for key, value in kwargs.items(): - try: - value = trans.security.decode_id( value ) - kwargs[ key ] = value - except: - pass - - # - # Execute tool. - # - tool = trans.app.toolbox.get_tool( tool_id ) - if not tool: - return messages.NO_TOOL - - # HACK: add run button so that tool.handle_input will run tool. - kwargs['runtool_btn'] = 'Execute' - params = util.Params( kwargs, sanitize = False ) - template, vars = tool.handle_input( trans, params.__dict__, history=target_history ) - - # TODO: check for errors and ensure that output dataset is available. - output_datasets = vars[ 'out_data' ].values() - return self.add_track_async( trans, output_datasets[0].id ) - - @web.expose - def rerun_tool( self, trans, dataset_id, tool_id, chrom=None, low=None, high=None, **kwargs ): - """ - Rerun a tool to produce a new output dataset that corresponds to a - dataset that a user is currently viewing. - """ - - # - # TODO: refactor to use same code as run_tool. - # - - # Run tool on region if region is specificied. - run_on_region = False - if chrom and low and high: - run_on_region = True - low, high = int( low ), int( high ) - - # Dataset check. - original_dataset = self.get_dataset( trans, dataset_id, check_ownership=False, check_accessible=True ) - msg = self._check_dataset_state( trans, original_dataset ) - if msg: - return to_json_string( msg ) - - # - # Set tool parameters--except non-hidden dataset parameters--using combination of - # job's previous parameters and incoming parameters. Incoming parameters - # have priority. - # - original_job = get_dataset_job( original_dataset ) - tool = trans.app.toolbox.get_tool( original_job.tool_id ) - if not tool: - return messages.NO_TOOL - tool_params = dict( [ ( p.name, p.value ) for p in original_job.parameters ] ) - # TODO: need to handle updates to conditional parameters; conditional - # params are stored in dicts (and dicts within dicts). - tool_params.update( dict( [ ( key, value ) for key, value in kwargs.items() if key in tool.inputs ] ) ) - tool_params = tool.params_from_strings( tool_params, self.app ) - - # - # If running tool on region, convert input datasets (create indices) so - # that can regions of data can be quickly extracted. - # - messages_list = [] - if run_on_region: - for jida in original_job.input_datasets: - input_dataset = jida.dataset - if get_data_provider( original_dataset=input_dataset ): - # Can index dataset. - track_type, data_sources = input_dataset.datatype.get_track_type() - # Convert to datasource that provides 'data' because we need to - # extract the original data. - data_source = data_sources[ 'data' ] - msg = self._convert_dataset( trans, input_dataset, data_source ) - if msg is not None: - messages_list.append( msg ) - - # Return any messages generated during conversions. - return_message = _get_highest_priority_msg( messages_list ) - if return_message: - return to_json_string( return_message ) - - # - # Set target history (the history that tool will use for inputs/outputs). - # If user owns dataset, put new data in original dataset's history; if - # user does not own dataset (and hence is accessing dataset via sharing), - # put new data in user's current history. - # - if original_dataset.history.user == trans.user: - target_history = original_dataset.history - else: - target_history = trans.get_history( create=True ) - hda_permissions = trans.app.security_agent.history_get_default_permissions( target_history ) - - def set_param_value( param_dict, param_name, param_value ): - """ - Set new parameter value in a tool's parameter dictionary. - """ - - # Recursive function to set param value. - def set_value( param_dict, group_name, group_index, param_name, param_value ): - if group_name in param_dict: - param_dict[ group_name ][ group_index ][ param_name ] = param_value - return True - elif param_name in param_dict: - param_dict[ param_name ] = param_value - return True - else: - # Recursive search. - return_val = False - for name, value in param_dict.items(): - if isinstance( value, dict ): - return_val = set_value( value, group_name, group_index, param_name, param_value) - if return_val: - return return_val - return False - - # Parse parameter name if necessary. - if param_name.find( "|" ) == -1: - # Non-grouping parameter. - group_name = group_index = None - else: - # Grouping parameter. - group, param_name = param_name.split( "|" ) - index = group.rfind( "_" ) - group_name = group[ :index ] - group_index = int( group[ index + 1: ] ) - - return set_value( param_dict, group_name, group_index, param_name, param_value ) - - # Set parameters based tool's trackster config. - params_set = {} - for action in tool.trackster_conf.actions: - success = False - for joda in original_job.output_datasets: - if joda.name == action.output_name: - set_param_value( tool_params, action.name, joda.dataset ) - params_set[ action.name ] = True - success = True - break - if not success: - return messages.ERROR - - # - # Set input datasets for tool. If running on region, extract and use subset - # when possible. - # - location = "%s:%i-%i" % ( chrom, low, high ) - for jida in original_job.input_datasets: - # If param set previously by config actions, do nothing. - if jida.name in params_set: - continue - - input_dataset = jida.dataset - if input_dataset is None: #optional dataset and dataset wasn't selected - tool_params[ jida.name ] = None - elif run_on_region and hasattr( input_dataset.datatype, 'get_track_type' ): - # Dataset is indexed and hence a subset can be extracted and used - # as input. - - # Look for subset. - subset_dataset_association = trans.sa_session.query( trans.app.model.HistoryDatasetAssociationSubset ) \ - .filter_by( hda=input_dataset, location=location ) \ - .first() - if subset_dataset_association: - # Data subset exists. - subset_dataset = subset_dataset_association.subset - else: - # Need to create subset. - track_type, data_sources = input_dataset.datatype.get_track_type() - data_source = data_sources[ 'data' ] - converted_dataset = input_dataset.get_converted_dataset( trans, data_source ) - deps = input_dataset.get_converted_dataset_deps( trans, data_source ) - - # Create new HDA for input dataset's subset. - new_dataset = trans.app.model.HistoryDatasetAssociation( extension=input_dataset.ext, \ - dbkey=input_dataset.dbkey, \ - create_dataset=True, \ - sa_session=trans.sa_session, - name="Subset [%s] of data %i" % \ - ( location, input_dataset.hid ), - visible=False ) - target_history.add_dataset( new_dataset ) - trans.sa_session.add( new_dataset ) - trans.app.security_agent.set_all_dataset_permissions( new_dataset.dataset, hda_permissions ) - - # Write subset of data to new dataset - data_provider_class = get_data_provider( original_dataset=input_dataset ) - data_provider = data_provider_class( original_dataset=input_dataset, - converted_dataset=converted_dataset, - dependencies=deps ) - trans.app.object_store.create( new_dataset.dataset ) - data_provider.write_data_to_file( chrom, low, high, new_dataset.file_name ) - - # TODO: (a) size not working; (b) need to set peek. - new_dataset.set_size() - new_dataset.info = "Data subset for trackster" - new_dataset.set_dataset_state( trans.app.model.Dataset.states.OK ) - - # Set metadata. - # TODO: set meta internally if dataset is small enough? - if trans.app.config.set_metadata_externally: - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, - trans, incoming = { 'input1':new_dataset }, - overwrite=False, job_params={ "source" : "trackster" } ) - else: - message = 'Attributes updated' - new_dataset.set_meta() - new_dataset.datatype.after_setting_metadata( new_dataset ) - - # Add HDA subset association. - subset_association = trans.app.model.HistoryDatasetAssociationSubset( hda=input_dataset, subset=new_dataset, location=location ) - trans.sa_session.add( subset_association ) - - subset_dataset = new_dataset - - trans.sa_session.flush() - - # Add dataset to tool's parameters. - if not set_param_value( tool_params, jida.name, subset_dataset ): - return to_json_string( { "error" : True, "message" : "error setting parameter %s" % jida.name } ) - - # - # Execute tool and handle outputs. - # - try: - subset_job, subset_job_outputs = tool.execute( trans, incoming=tool_params, - history=target_history, - job_params={ "source" : "trackster" } ) - except Exception, e: - # Lots of things can go wrong when trying to execute tool. - return to_json_string( { "error" : True, "message" : e.__class__.__name__ + ": " + str(e) } ) - if run_on_region: - for output in subset_job_outputs.values(): - output.visible = False - trans.sa_session.flush() - - # - # Return new track that corresponds to the original dataset. - # - output_name = None - for joda in original_job.output_datasets: - if joda.dataset == original_dataset: - output_name = joda.name - break - for joda in subset_job.output_datasets: - if joda.name == output_name: - output_dataset = joda.dataset - - return self.add_track_async( trans, output_dataset.id ) - @web.expose @web.require_login( "use Galaxy visualizations", use_panels=True ) def paramamonster( self, trans, hda_ldda, dataset_id ): @@ -799,18 +514,6 @@ # Helper methods. # ----------------- - def _check_dataset_state( self, trans, dataset ): - """ - Returns a message if dataset is not ready to be used in visualization. - """ - if not dataset: - return messages.NO_DATA - if dataset.state == trans.app.model.Job.states.ERROR: - return messages.ERROR - if dataset.state != trans.app.model.Job.states.OK: - return messages.PENDING - return None - def _get_datasources( self, trans, dataset ): """ Returns datasources for dataset; if datasources are not available @@ -833,56 +536,10 @@ data_sources_dict[ source_type ] = { "name" : data_source, "message": msg } return data_sources_dict - - def _convert_dataset( self, trans, dataset, target_type ): - """ - Converts a dataset to the target_type and returns a message indicating - status of the conversion. None is returned to indicate that dataset - was converted successfully. - """ - - # Get converted dataset; this will start the conversion if necessary. - try: - converted_dataset = dataset.get_converted_dataset( trans, target_type ) - except NoConverterException: - return messages.NO_CONVERTER - except ConverterDependencyException, dep_error: - return { 'kind': messages.ERROR, 'message': dep_error.value } - - # Check dataset state and return any messages. - msg = None - if converted_dataset and converted_dataset.state == model.Dataset.states.ERROR: - job_id = trans.sa_session.query( trans.app.model.JobToOutputDatasetAssociation ) \ - .filter_by( dataset_id=converted_dataset.id ).first().job_id - job = trans.sa_session.query( trans.app.model.Job ).get( job_id ) - msg = { 'kind': messages.ERROR, 'message': job.stderr } - elif not converted_dataset or converted_dataset.state != model.Dataset.states.OK: - msg = messages.PENDING - - return msg - + def _get_dataset( self, trans, hda_ldda, dataset_id ): """ Returns either HDA or LDDA for hda/ldda and id combination. """ if hda_ldda == "hda": return self.get_dataset( trans, dataset_id, check_ownership=False, check_accessible=True ) else: - return trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id( dataset_id ) ) - - -def _get_highest_priority_msg( message_list ): - """ - Returns highest priority message from a list of messages. - """ - return_message = None - - # For now, priority is: job error (dict), no converter, pending. - for message in message_list: - if message is not None: - if isinstance(message, dict): - return_message = message - break - elif message == messages.NO_CONVERTER: - return_message = message - elif return_message == None and message == messages.PENDING: - return_message = message - return return_message + return trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id( dataset_id ) ) \ No newline at end of file diff -r b2eabe39a70f676b8cb3b90a656501804547fd87 -r 4fbd05095ca70adf740ed635451c0ec876635f50 lib/galaxy/web/controllers/visualization.py --- a/lib/galaxy/web/controllers/visualization.py +++ b/lib/galaxy/web/controllers/visualization.py @@ -69,7 +69,7 @@ class VisualizationController( BaseUIController, SharableMixin, UsesAnnotations, - UsesHistoryMixinDatasetAssociationMixin, UsesVisualizationMixin, + UsesHistoryDatasetAssociationMixin, UsesVisualizationMixin, UsesItemRatings ): _user_list_grid = VisualizationListGrid() _published_list_grid = VisualizationAllPublishedGrid() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.