1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3189a1bf18af/ changeset: 3189a1bf18af user: dan date: 2013-02-04 12:33:19 summary: Add sentry_dsn to Tool Shed config.py' lib/galaxy/webapps/community/config.py affected #: 15 files diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -59,6 +59,7 @@ <datatype extension="bowtie_base_index" type="galaxy.datatypes.ngsindex:BowtieBaseIndex" mimetype="text/html" display_in_upload="False"/><datatype extension="csfasta" type="galaxy.datatypes.sequence:csFasta" display_in_upload="true"/><datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream" max_optional_metadata_filesize="1048576" /> + <datatype extension="data_manager_json" type="galaxy.datatypes.data:Text" mimetype="application/json" subclass="True" display_in_upload="False"/><datatype extension="fasta" type="galaxy.datatypes.sequence:Fasta" display_in_upload="true"><converter file="fasta_to_tabular_converter.xml" target_datatype="tabular"/><converter file="fasta_to_bowtie_base_index_converter.xml" target_datatype="bowtie_base_index"/> diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -18,6 +18,7 @@ from galaxy.tools.genome_index import load_genome_index_tools from galaxy.sample_tracking import external_service_types from galaxy.openid.providers import OpenIDProviders +from galaxy.tools.data_manager.manager import DataManagers class UniverseApplication( object ): """Encapsulates the state of a Universe application""" @@ -93,6 +94,8 @@ self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) # Search support for tools self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) + #datamanager + self.data_managers = DataManagers( self ) # If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories. if self.config.get_bool( 'enable_tool_shed_check', False ): from tool_shed import update_manager diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -75,6 +75,10 @@ except: self.hours_between_check = 12 self.update_integrated_tool_panel = kwargs.get( "update_integrated_tool_panel", True ) + self.enable_data_manager_user_view = string_as_bool( kwargs.get( "enable_data_manager_user_view", "False" ) ) + self.data_manager_config_file = resolve_path( kwargs.get(' data_manager_config_file', 'data_manager_conf.xml' ), self.root ) + self.data_manager_move_files = string_as_bool( kwargs.get( "data_manager_move_files", "False" ) ) + self.galaxy_data_manager_data_path = kwargs.get( 'galaxy_data_manager_data_dir', self.tool_data_path ) self.tool_secret = kwargs.get( "tool_secret", "" ) self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) ) diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -44,7 +44,7 @@ log.debug( "%d workers ready", nworkers ) def run_next( self ): - """Run the next job, waiting until one is available if neccesary""" + """Run the next job, waiting until one is available if necessary""" while 1: job_wrapper = self.queue.get() if job_wrapper is self.STOP_SIGNAL: diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2996,6 +2996,20 @@ def set_item( self, visualization ): self.visualization = visualization +#Data Manager Classes +class DataManagerHistoryAssociation( object ): + def __init__( self, id=None, history=None, user=None ): + self.id = id + self.history = history + self.user = user + +class DataManagerJobAssociation( object ): + def __init__( self, id=None, job=None, data_manager_id=None ): + self.id = id + self.job = job + self.data_manager_id = data_manager_id +#end of Data Manager Classes + class UserPreference ( object ): def __init__( self, name=None, value=None ): self.name = name diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -930,6 +930,23 @@ Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) +#Data Manager tables +DataManagerHistoryAssociation.table = Table( "data_manager_history_association", metadata, + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) + ) + +DataManagerJobAssociation.table = Table( "data_manager_job_association", metadata, + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), + Column( "data_manager_id", TEXT, index=True ) + ) + # Tagging tables. Tag.table = Table( "tag", metadata, @@ -1899,6 +1916,17 @@ properties=dict( visualization=relation( Visualization ), user=relation( User ) ) ) +#Data Manager tables +assign_mapper( context, DataManagerHistoryAssociation, DataManagerHistoryAssociation.table, + properties=dict( history=relation( History ), + user=relation( User, backref='data_manager_histories' ) + ) + ) + +assign_mapper( context, DataManagerJobAssociation, DataManagerJobAssociation.table, + properties=dict( job=relation( Job, backref=backref('data_manager_association', uselist=False ), uselist=False ) ) + ) + # User tables. assign_mapper( context, UserPreference, UserPreference.table, diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -6,7 +6,7 @@ pkg_resources.require( "simplejson" ) pkg_resources.require( "Mako" ) -import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess, random, math, traceback, re +import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess, random, math, traceback, re, pipes import simplejson import binascii from mako.template import Template @@ -25,6 +25,7 @@ from galaxy.util.expressions import ExpressionContext from galaxy.tools.test import ToolTestBuilder from galaxy.tools.actions import DefaultToolAction +from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.deps import DependencyManager from galaxy.model import directory_hash_id from galaxy.model.orm import * @@ -79,6 +80,7 @@ # In-memory dictionary that defines the layout of the tool panel. self.tool_panel = odict() self.index = 0 + self.data_manager_tools = odict() # File that contains the XML section and tool tags from all tool panel config files integrated into a # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. @@ -508,7 +510,7 @@ self.integrated_tool_panel[ key ] = integrated_section else: self.integrated_tool_panel.insert( index, key, integrated_section ) - def load_tool( self, config_file, guid=None ): + def load_tool( self, config_file, guid=None, **kwds ): """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" # Parse XML configuration file and get the root element tree = util.parse_xml( config_file ) @@ -524,7 +526,7 @@ ToolClass = tool_types.get( root.get( 'tool_type' ) ) else: ToolClass = Tool - return ToolClass( config_file, root, self.app, guid=guid ) + return ToolClass( config_file, root, self.app, guid=guid, **kwds ) def reload_tool_by_id( self, tool_id ): """ Attempt to reload the tool identified by 'tool_id', if successful @@ -811,6 +813,7 @@ """ tool_type = 'default' + default_tool_action = DefaultToolAction def __init__( self, config_file, root, app, guid=None ): """Load a tool from the config named by `config_file`""" @@ -1051,7 +1054,7 @@ # Action action_elem = root.find( "action" ) if action_elem is None: - self.tool_action = DefaultToolAction() + self.tool_action = self.default_tool_action() else: module = action_elem.get( 'module' ) cls = action_elem.get( 'class' ) @@ -2570,18 +2573,24 @@ temp_file_path = os.path.join( job_working_directory, "dataset_%s_files" % ( hda.dataset.id ) ) try: a_files = os.listdir( temp_file_path ) + print 'a_files',a_files if len( a_files ) > 0: for f in a_files: + print 'f', f self.app.object_store.update_from_file(hda.dataset, extra_dir="dataset_%d_files" % hda.dataset.id, alt_name = f, file_name = os.path.join(temp_file_path, f), - create = True) + create = True, + preserve_symlinks = True ) # Clean up after being handled by object store. # FIXME: If the object (e.g., S3) becomes async, this will # cause issues so add it to the object store functionality? + print 'before rmtree' shutil.rmtree(temp_file_path) - except: + print 'after rm tree' + except Exception, e: + log.debug( "Error in collect_associated_files: %s" % ( e ) ) continue def collect_child_datasets( self, output, job_working_directory ): """ @@ -2806,7 +2815,64 @@ return tool_dict -class DataSourceTool( Tool ): + def get_default_history_by_trans( self, trans, create=False ): + return trans.get_history( create=create ) + + +class OutputParameterJSONTool( Tool ): + """ + Alternate implementation of Tool that provides parameters and other values + JSONified within the contents of an output dataset + """ + tool_type = 'output_parameter_json' + def _prepare_json_list( self, param_list ): + rval = [] + for value in param_list: + if isinstance( value, dict ): + rval.append( self._prepare_json_param_dict( value ) ) + elif isinstance( value, list ): + rval.append( self._prepare_json_list( value ) ) + else: + rval.append( str( value ) ) + return rval + def _prepare_json_param_dict( self, param_dict ): + rval = {} + for key, value in param_dict.iteritems(): + if isinstance( value, dict ): + rval[ key ] = self._prepare_json_param_dict( value ) + elif isinstance( value, list ): + rval[ key ] = self._prepare_json_list( value ) + else: + rval[ key ] = str( value ) + return rval + def exec_before_job( self, app, inp_data, out_data, param_dict=None ): + if param_dict is None: + param_dict = {} + json_params = {} + json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) #it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? + json_params[ 'output_data' ] = [] + json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) + json_filename = None + for i, ( out_name, data ) in enumerate( out_data.iteritems() ): + #use wrapped dataset to access certain values + wrapped_data = param_dict.get( out_name ) + #allow multiple files to be created + file_name = str( wrapped_data ) + extra_files_path = str( wrapped_data.files_path ) + data_dict = dict( out_data_name = out_name, + ext = data.ext, + dataset_id = data.dataset.id, + hda_id = data.id, + file_name = file_name, + extra_files_path = extra_files_path ) + json_params[ 'output_data' ].append( data_dict ) + if json_filename is None: + json_filename = file_name + out = open( json_filename, 'w' ) + out.write( simplejson.dumps( json_params ) ) + out.close() + +class DataSourceTool( OutputParameterJSONTool ): """ Alternate implementation of Tool for data_source tools -- those that allow the user to query and extract data from another web site. @@ -2816,29 +2882,10 @@ def _build_GALAXY_URL_parameter( self ): return ToolParameter.build( self, ElementTree.XML( '<param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=%s" />' % self.id ) ) def parse_inputs( self, root ): - Tool.parse_inputs( self, root ) + super( DataSourceTool, self ).parse_inputs( root ) if 'GALAXY_URL' not in self.inputs: self.inputs[ 'GALAXY_URL' ] = self._build_GALAXY_URL_parameter() - def _prepare_datasource_json_list( self, param_list ): - rval = [] - for value in param_list: - if isinstance( value, dict ): - rval.append( self._prepare_datasource_json_param_dict( value ) ) - elif isinstance( value, list ): - rval.append( self._prepare_datasource_json_list( value ) ) - else: - rval.append( str( value ) ) - return rval - def _prepare_datasource_json_param_dict( self, param_dict ): - rval = {} - for key, value in param_dict.iteritems(): - if isinstance( value, dict ): - rval[ key ] = self._prepare_datasource_json_param_dict( value ) - elif isinstance( value, list ): - rval[ key ] = self._prepare_datasource_json_list( value ) - else: - rval[ key ] = str( value ) - return rval + self.inputs_by_page[0][ 'GALAXY_URL' ] = self.inputs[ 'GALAXY_URL' ] def exec_before_job( self, app, inp_data, out_data, param_dict=None ): if param_dict is None: param_dict = {} @@ -2848,7 +2895,7 @@ name = param_dict.get( 'name' ) json_params = {} - json_params[ 'param_dict' ] = self._prepare_datasource_json_param_dict( param_dict ) #it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? + json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) #it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params[ 'output_data' ] = [] json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None @@ -2939,9 +2986,186 @@ class GenomeIndexTool( Tool ): tool_type = 'index_genome' +class DataManagerTool( OutputParameterJSONTool ): + tool_type = 'manage_data' + default_tool_action = DataManagerToolAction + + def __init__( self, config_file, root, app, guid=None, data_manager_id=None, **kwds ): + self.data_manager_id = data_manager_id + super( DataManagerTool, self ).__init__( config_file, root, app, guid=guid, **kwds ) + if self.data_manager_id is None: + self.data_manager_id = self.id + + #def parse_inputs( self, root ): + # super( DataManagerTool, self ).parse_inputs( root ) + # ''' + # if '__GALAXY_MOVE_OUTPUT_FILES__' not in self.inputs: + # self.inputs[ '__GALAXY_MOVE_OUTPUT_FILES__' ] = ToolParameter.build( self, ElementTree.XML( '<param name="__GALAXY_MOVE_OUTPUT_FILES__" label="Move created data to cache destination" type="boolean" truevalue="MOVE" falsevalue="DO_NOT_MOVE" checked="%s" />' % self.app.config.data_manager_move_files ) ) + # print 'self.inputs_by_page',self.inputs_by_page + # self.inputs_by_page[0][ '__GALAXY_MOVE_OUTPUT_FILES__' ] = self.inputs[ '__GALAXY_MOVE_OUTPUT_FILES__' ] + # print 'self.inputs', self.inputs + # ''' + # #self.inputs[ '__DATA_MANAGER_ID__' ] = ToolParameter.build( self, ElementTree.XML( '<param name="__DATA_MANAGER_ID__" type="hidden" value="%s" />' % ( self.data_manager_id ) ) ) + # #self.inputs_by_page[0][ '__DATA_MANAGER_ID__' ] = self.inputs[ '__DATA_MANAGER_ID__' ] + + def exec_after_process( self, app, inp_data, out_data, param_dict, job = None, **kwds ): + #run original exec_after_process + super( DataManagerTool, self ).exec_after_process( app, inp_data, out_data, param_dict, job = job, **kwds ) + #process results of tool + print 'exect after', self.id + print 'inp_data', inp_data + print 'out_data', out_data + print 'param_dict', param_dict + print 'job', job, job.state + if job and job.state == job.states.ERROR: + return + #print 'data_manager.output_ref',data_manager.output_ref + #data_manager = self.app.data_managers.get( self.id, None ) #fix me to not only use tool ID! + data_manager_id = job.data_manager_association.data_manager_id + data_manager = self.app.data_managers.get( data_manager_id, None ) + #TODO: need to be able to handle using a data manager tool for more than one manager + #manager id is currently same as tool id + assert data_manager is not None, "Invalid data manager (%s) requested. It may have been removed before the job completed." % ( data_manager_id ) + data_manager_dicts = {} + data_manager_dict = {} + #TODO: fix this merging below + for output_name, output_dataset in out_data.iteritems(): + try: + output_dict = simplejson.loads( open( output_dataset.file_name ).read() ) + except Exception, e: + log.warning( 'Error reading DataManagerTool json for "%s": %s' % ( output_name, e ) ) + continue + data_manager_dicts[ output_name ] = output_dict + print 'data_manager_dicts', data_manager_dicts + for key, value in output_dict.iteritems(): + if key not in data_manager_dict: + data_manager_dict[ key ] = {} + print 'key', key + print ' data_manager_dict[ key ]', data_manager_dict[ key ] + print 'value', value + data_manager_dict[ key ].update( value ) + data_manager_dict.update( output_dict ) + + print 'data_manager_dicts',data_manager_dicts + print 'data_manager_dict', data_manager_dict + data_tables_dict = data_manager_dict.get( 'data_tables', {} ) + #for data_table_name, data_table_values in data_tables_dict.iteritems(): + for data_table_name, data_table_columns in data_manager.data_tables.iteritems(): + print 'data_table_name', data_table_name + data_table_values = data_tables_dict.pop( data_table_name, None ) #data_tables_dict.get( data_table_name, [] ) + if not data_table_values: + log.warning( 'No values for data table "%s" were returned by the data manager "%s".' % ( data_table_name, data_manager.id ) ) + continue #next data table + data_table = app.tool_data_tables.get( data_table_name, None ) + if data_table is None: + log.error( 'The data manager "%s" returned an unknown data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( data_manager.id, data_table_name, data_table_values, data_table_name, 'tool_data_table_conf.xml' ) ) + continue #next table name + output_ref_values = {} + if data_table_name in data_manager.output_ref_by_data_table: + for data_table_column, output_ref in data_manager.output_ref_by_data_table[ data_table_name ].iteritems(): + output_ref_dataset = out_data.get( output_ref, None ) + assert output_ref_dataset is not None, "Referenced output was not found." + output_ref_values[ data_table_column ] = output_ref_dataset + print 'output_ref_values', output_ref_values + + final_data_table_values = [] + if not isinstance( data_table_values, list ): + data_table_values = [ data_table_values ] + columns = data_table.get_column_name_list() + + try: + data_table_fh = open( data_table.filename, 'r+b' ) + except IOError, e: + log.warning( 'Error opening data table file (%s) with r+b, assuming file does not exist and will open as wb: %s' % ( data_table.filename, e ) ) + data_table_fh = open( data_table.filename, 'wb' ) + if os.stat( data_table.filename )[6] != 0: + # ensure last existing line ends with new line + data_table_fh.seek( -1, 2 ) #last char in file + last_char = data_table_fh.read() + if last_char not in [ '\n', '\r' ]: + data_table_fh.write( '\n' ) + for data_table_row in data_table_values: + data_table_value = dict( **data_table_row ) #keep original values here + for name, value in data_table_row.iteritems(): #FIXME: need to loop through here based upon order listed in data_manager config + if name in output_ref_values: + #TODO: Allow moving! + #if param_dict[ '__GALAXY_MOVE_OUTPUT_FILES__' ]: + # #FIXME: allow moving + # log.error( "\n\nShould be moving output files directory, but not implemented yet.\n" ) + # base_path = output_ref_values[ name ].extra_files_path + #else: + # base_path = output_ref_values[ name ].extra_files_path + moved = data_manager.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value ) + print 'moved', moved #should we always move? + data_table_value[ name ] = data_manager.process_value_translation( data_table_name, name, **data_table_value ) + final_data_table_values.append( data_table_value ) + fields = [] + for column_name in columns: + if column_name is None or column_name not in data_table_value: + fields.append( data_table.get_empty_field_by_name( column_name ) ) + else: + fields.append( data_table_value[ column_name ] ) + print 'fields', fields + #should we add a comment to file about automatically generated value here? + data_table_fh.write( "%s\n" % ( data_table.separator.join( self._replace_field_separators( fields, separator=data_table.separator ) ) ) ) #write out fields to disk + data_table.data.append( fields ) #add fields to loaded data table + print 'final_data_table_values', final_data_table_values + print 'data_table.data', data_table.data + data_table_fh.close() + for data_table_name, data_table_values in data_tables_dict.iteritems(): + #tool returned extra data table entries, but data table was not declared in data manager + #do not add these values, but do provide messages + log.warning( 'The data manager "%s" returned an undeclared data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( data_manager.id, data_table_name, data_table_values, data_table_name, self.app.data_managers.filename ) ) + + def _replace_field_separators( self, fields, separator="\t", replace=None, comment_char=None ): + #make sure none of the fields contain separator + #make sure separator replace is different from comment_char, + #due to possible leading replace + if replace is None: + if separator == " ": + if comment_char == "\t": + replace = "_" + else: + replace = "\t" + else: + if comment_char == " ": + replace = "_" + else: + replace = " " + return map( lambda x: x.replace( separator, replace ), fields ) + + def get_default_history_by_trans( self, trans, create=False ): + def _create_data_manager_history( user ): + history = trans.app.model.History( name='Data Manager History (automatically created)', user=user ) + data_manager_association = trans.app.model.DataManagerHistoryAssociation( user=user, history=history ) + trans.sa_session.add_all( ( history, data_manager_association ) ) + trans.sa_session.flush() + return history + user = trans.user + assert user, 'You must be logged in to use this tool.' + history = user.data_manager_histories + if not history: + #create + if create: + history = _create_data_manager_history( user ) + else: + history = None + else: + for history in reversed( history ): + history = history.history + if not history.deleted: + break + if history.deleted: + if create: + history = _create_data_manager_history( user ) + else: + history = None + return history + + # Populate tool_type to ToolClass mappings tool_types = {} -for tool_class in [ Tool, DataDestinationTool, SetMetadataTool, DataSourceTool, AsyncDataSourceTool ]: +for tool_class in [ Tool, DataDestinationTool, SetMetadataTool, DataSourceTool, AsyncDataSourceTool, DataManagerTool ]: tool_types[ tool_class.tool_type ] = tool_class # ---- Utility classes to be factored out ----------------------------------- @@ -2983,6 +3207,12 @@ """ def __nonzero__( self ): return bool( self.value ) + def get_display_text( self, quote=True ): + print 'self.input',self.input + print 'self.input.tool.app', self.input.tool.app + print 'self.value', self.value + print 'self.input.value_to_display_text( self.value, self.input.tool.app )', self.input.value_to_display_text( self.value, self.input.tool.app ) + return pipes.quote( self.input.value_to_display_text( self.value, self.input.tool.app ) ) class RawObjectWrapper( ToolParameterValueWrapper ): """ diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -168,7 +168,7 @@ # Set history. if not history: - history = trans.history + history = tool.get_default_history_by_trans( trans, create=True ) #trans..history out_data = odict() # Collect any input datasets from the incoming parameters diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/tools/data/__init__.py --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -28,6 +28,11 @@ return self.data_tables.__getitem__( key ) def __contains__( self, key ): return self.data_tables.__contains__( key ) + def get( self, name, default=None ): + try: + return self[ name ] + except KeyError: + return default def load_from_config_file( self, config_filename, tool_data_path, from_shed_config=False ): """ This method is called under 3 conditions: @@ -125,6 +130,8 @@ def __init__( self, config_element, tool_data_path ): self.name = config_element.get( 'name' ) self.comment_char = config_element.get( 'comment_char' ) + self.empty_field_value = config_element.get( 'empty_field_value', '' ) + self.empty_field_values = {} for file_elem in config_element.findall( 'file' ): # There should only be one file_elem. if 'path' in file_elem.attrib: @@ -134,6 +141,8 @@ self.tool_data_file = None self.tool_data_path = tool_data_path self.missing_index_file = None + def get_empty_field_by_name( self, name ): + return self.empty_field_values.get( name, self.empty_field_value ) class TabularToolDataTable( ToolDataTable ): """ @@ -176,6 +185,7 @@ if os.path.exists( filename ): found = True all_rows.extend( self.parse_file_fields( open( filename ) ) ) + self.filename = filename else: # Since the path attribute can include a hard-coded path to a specific directory # (e.g., <file path="tool-data/cg_crr_files.loc" />) which may not be the same value @@ -187,6 +197,7 @@ if os.path.exists( corrected_filename ): found = True all_rows.extend( self.parse_file_fields( open( corrected_filename ) ) ) + self.filename = corrected_filename if not found: self.missing_index_file = filename log.warn( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) ) @@ -222,6 +233,9 @@ self.columns[name] = index if index > self.largest_index: self.largest_index = index + empty_field_value = column_elem.get( 'empty_field_value', None ) + if empty_field_value is not None: + self.empty_field_values[ name ] = empty_field_value assert 'value' in self.columns, "Required 'value' column missing from column def" if 'name' not in self.columns: self.columns['name'] = self.columns['value'] @@ -240,7 +254,19 @@ fields = line.split( self.separator ) if self.largest_index < len( fields ): rval.append( fields ) - return rval + return rval + def get_column_name_list( self ): + rval = [] + for i in range( self.largest_index + 1 ): + found_column = False + for name, index in self.columns.iteritems(): + if index == i: + rval.append( name ) + found_column = True + break + if not found_column: + rval.append( None ) + return rval # Registry of tool data types by type_key tool_data_table_types = dict( [ ( cls.type_key, cls ) for cls in [ TabularToolDataTable ] ] ) diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -880,6 +880,9 @@ >>> print p.filter_value( "hg17" ) hg17 """ + def __init__( self, *args, **kwds ): + super( GenomeBuildParameter, self ).__init__( *args, **kwds ) + self.static_options = [ ( value, key, False ) for key, value in util.dbnames ] def get_options( self, trans, other_values ): if not trans.history: yield 'unspecified', '?', False diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -567,6 +567,22 @@ return curdir return join( *rel_list ) +def relativize_symlinks( path, start=None, followlinks=False): + for root, dirs, files in os.walk( path, followlinks=followlinks ): + rel_start = None + for file_name in files: + symlink_file_name = os.path.join( root, file_name ) + if os.path.islink( symlink_file_name ): + symlink_target = os.readlink( symlink_file_name ) + if rel_start is None: + if start is None: + rel_start = root + else: + rel_start = start + rel_path = relpath( symlink_target, rel_start ) + os.remove( symlink_file_name ) + os.symlink( rel_path, symlink_file_name ) + def stringify_dictionary_keys( in_dict ): #returns a new dictionary #changes unicode keys into strings, only works on top level (does not recurse) diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/webapps/community/config.py --- a/lib/galaxy/webapps/community/config.py +++ b/lib/galaxy/webapps/community/config.py @@ -92,6 +92,8 @@ self.job_handlers = [] self.tool_handlers = [] self.tool_runners = [] + # Error logging with sentry + self.sentry_dsn = kwargs.get( 'sentry_dsn', None ) # Where the tool shed hgweb.config file is stored - the default is the Galaxy installation directory. self.hgweb_config_dir = resolve_path( kwargs.get( 'hgweb_config_dir', '' ), self.root ) # Proxy features diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 lib/galaxy/webapps/galaxy/controllers/tool_runner.py --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -89,7 +89,8 @@ tool.input_translator.translate( params ) # We may be visiting Galaxy for the first time ( e.g., sending data from UCSC ), # so make sure to create a new history if we've never had one before. - history = trans.get_history( create=True ) + #history = trans.get_history( create=True ) + history = tool.get_default_history_by_trans( trans, create=True ) template, vars = tool.handle_input( trans, params.__dict__ ) if len( params ) > 0: trans.log_event( "Tool params: %s" % ( str( params ) ), tool_id=tool_id ) diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 templates/webapps/galaxy/admin/index.mako --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -57,8 +57,10 @@ <div class="toolTitle"><a href="${h.url_for( controller='admin', action='quotas' )}" target="galaxy_main">Manage quotas</a></div><div class="toolTitle"><a href="${h.url_for( controller='library_admin', action='browse_libraries' )}" target="galaxy_main">Manage data libraries</a></div> %if trans.app.config.enable_beta_job_managers: - <div class="toolTitle"><a href="${h.url_for( controller='data_admin', action='manage_data' )}" target="galaxy_main">Manage local data</a></div> + <div class="toolTitle"><a href="${h.url_for( controller='data_admin', action='manage_data' )}" target="galaxy_main">Manage old local data</a></div> %endif + ##how to name this? + <div class="toolTitle"><a href="${h.url_for( controller='data_manager' )}" target="galaxy_main">Manage local (cached) data (beta)</a></div></div></div><div class="toolSectionPad"></div> diff -r d7f4d3a8d0b273163ae3d04872e211b7dcfbeba9 -r 3189a1bf18af72f750e476caa54d7e3ae2d3cc36 tools/data_source/upload.xml --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -33,7 +33,7 @@ </param><param name="async_datasets" type="hidden" value="None"/><upload_dataset name="files" title="Specify Files for Dataset" file_type_name="file_type" metadata_ref="files_metadata"> - <param name="file_data" type="file" size="30" label="File" ajax-upload="true" help="TIP: Due to browser limitations, uploading files larger than 2GB is guaranteed to fail. To upload large files, use the URL method (below) or FTP (if enabled by the site administrator)."> + <param name="file_data" type="file" size="30" label="File" ajax-upload="False" help="TIP: Due to browser limitations, uploading files larger than 2GB is guaranteed to fail. To upload large files, use the URL method (below) or FTP (if enabled by the site administrator)."><validator type="expression" message="You will need to reselect the file you specified (%s)." substitute_value_in_message="True">not ( ( isinstance( value, unicode ) or isinstance( value, str ) ) and value != "" )</validator><!-- use validator to post message to user about needing to reselect the file, since most browsers won't accept the value attribute for file inputs --></param><param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.