# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1280415308 14400 # Node ID d27912281a74ba74a8df6f5d72db7c553ca846c5 # Parent af48a13e46b9ab0136bcbf14508bd9bb044ad257 lims: - added a new table 'sample_dataset' to store sample datasets & their info when they are transfered from the sequencer - the datasets transfer page now uses a grid to facilitate bulk renaming - bulk renaming possible to fix the problem with the way SOLiD generates datasets - the remote file browser is now independent of a specific sample, the user may select any sample when transferring datasets from the sequencer --- a/templates/requests/common/sample_datasets.mako +++ b/templates/requests/common/sample_datasets.mako @@ -1,5 +1,5 @@ <%def name="render_sample_datasets( cntrller, sample )"> - <a href="${h.url_for(controller='requests_common', cntrller=cntrller, action='show_datatx_page', sample_id=trans.security.encode_id(sample.id))}">${sample.transferred_dataset_files()}/${len(sample.dataset_files)}</a> + <a href="${h.url_for(controller='requests_common', cntrller=cntrller, action='show_datatx_page', sample_id=trans.security.encode_id(sample.id))}">${sample.transferred_dataset_files()}/${len(sample.datasets)}</a></%def> --- /dev/null +++ b/templates/admin/requests/get_data.mako @@ -0,0 +1,144 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + + +<script type="text/javascript"> +$(document).ready(function(){ + //hide the all of the element with class msg_body + $(".msg_body").hide(); + //toggle the componenet with class msg_body + $(".msg_head").click(function(){ + $(this).next(".msg_body").slideToggle(450); + }); +}); + + + + +</script> + +<script type="text/javascript"> + function display_file_details(request_id, folder_path) + { + var w = document.get_data.files_list.selectedIndex; + var selected_value = document.get_data.files_list.options[w].value; + var cell = $("#file_details"); + if(selected_value.charAt(selected_value.length-1) != '/') + { + // Make ajax call + $.ajax( { + type: "POST", + url: "${h.url_for( controller='requests_admin', action='get_file_details' )}", + dataType: "json", + data: { id: request_id, folder_path: document.get_data.folder_path.value+selected_value }, + success : function ( data ) { + cell.html( '<label>'+data+'</label>' ) + } + }); + } + else + { + cell.html( '' ) + } + + + } +</script> + +<script type="text/javascript"> + function open_folder1(request_id, folder_path) + { + var w = document.get_data.files_list.selectedIndex; + var selected_value = document.get_data.files_list.options[w].value; + var cell = $("#file_details"); + if(selected_value.charAt(selected_value.length-1) == '/') + { + document.get_data.folder_path.value = document.get_data.folder_path.value+selected_value + // Make ajax call + $.ajax( { + type: "POST", + url: "${h.url_for( controller='requests_admin', action='open_folder' )}", + dataType: "json", + data: { id: request_id, folder_path: document.get_data.folder_path.value }, + success : function ( data ) { + document.get_data.files_list.options.length = 0 + for(i=0; i<data.length; i++) + { + var newOpt = new Option(data[i], data[i]); + document.get_data.files_list.options[i] = newOpt; + } + //cell.html( '<label>'+data+'</label>' ) + + } + }); + } + else + { + cell.html( '' ) + } + } +</script> + + +<style type="text/css"> +.msg_head { + padding: 0px 0px; + cursor: pointer; +} + +} +</style> + +%if message: + ${render_msg( message, status )} +%endif +<br/> +<br/> +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( controller='requests_admin', action='manage_request_types', operation='view', id=trans.security.encode_id(request.type.id) )}"> + <span>Sequencer information</span></a> + </li> + <li> + <a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='show', id=trans.security.encode_id(request.id) )}"> + <span>Browse this request</span></a> + </li> +</ul> + +<form name="get_data" id="get_data" action="${h.url_for( controller='requests_admin', cntrller=cntrller, action='get_data', request_id=request.id)}" method="post" > + <div class="toolFormTitle">Select files for transfer</div> + <div class="toolForm"> + <div class="form-row"> + <label>Sample:</label> + ${samples_selectbox.get_html()} + <div class="toolParamHelp" style="clear: both;"> + Select the sample with which you want to associate the dataset(s) + </div> + <br/> + <label>Folder path on the sequencer:</label> + <input type="text" name="folder_path" value="${folder_path}" size="100"/> + <input type="submit" name="browse_button" value="List contents"/> + ##<input type="submit" name="open_folder" value="Open folder"/> + <input type="submit" name="folder_up" value="Up"/> + </div> + <div class="form-row"> + <select name="files_list" id="files_list" style="max-width: 60%; width: 98%; height: 150px; font-size: 100%;" ondblclick="open_folder1(${request.id}, '${folder_path}')" onChange="display_file_details(${request.id}, '${folder_path}')" multiple> + %for index, f in enumerate(files): + <option value="${f}">${f}</option> + %endfor + </select> + <br/> + <div id="file_details" class="toolParamHelp" style="clear: both;"> + + </div> + </div> + <div class="form-row"> +<!-- <div class="toolParamHelp" style="clear: both;"> + After selecting dataset(s), be sure to click on the <b>Start transfer</b> button. + Once the transfer is complete the dataset(s) will show up on this page. + </div>--> + <input type="submit" name="select_show_datasets_button" value="Select & show datasets"/> + <input type="submit" name="select_more_button" value="Select more"/> + </div> + </div> +</form> --- /dev/null +++ b/templates/admin/requests/rename_datasets.mako @@ -0,0 +1,66 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, status )} +%endif +<h3>Rename datasets for Sample "${sample.name}"</h3> +<br/> +${render_msg('A dataset can be renamed only if its status is "Not Started"', 'ok')} + +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( controller='requests_admin', action='manage_datasets', sample_id=sample.id )}"> + <span>Browse datasets</span></a> + </li> + <li> + <a class="action-button" href="${h.url_for( controller='requests_admin', action='list', operation='show', id=trans.security.encode_id(sample.request.id) )}"> + <span>Browse this request</span></a> + </li> +</ul> + +<form name="rename_datasets" id="rename_datasets" action="${h.url_for( controller='requests_admin', action='rename_datasets', id_list=id_list, sample_id=trans.security.encode_id(sample.id))}" method="post" > + <table class="grid"> + <thead> + <tr> + <th>Prepend directory name</th> + <th>Name</th> + <th>Path on Sequencer</th> + </tr> + <thead> + <tbody> + %for id in id_list: + <% + sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id) ) + import os + id = trans.security.decode_id(id) + %> + %if sample_dataset.status == trans.app.model.Sample.transfer_status.NOT_STARTED: + <tr> + <td> + <select name="prepend_${sample_dataset.id}" last_selected_value="2"> + <option value="None" selected></option> + %for option_index, option in enumerate(sample_dataset.file_path.split(os.sep)[:-1]): + %if option.strip(): + <option value="${option}">${option}</option> + %endif + %endfor + </select> + </td> + <td> + <input type="text" name="name_${sample_dataset.id}" value="${sample_dataset.name}" size="100"/> + </td> + <td> + ${sample_dataset.file_path} + </td> + </tr> + %endif + %endfor + </tbody> + </table> + <br/> + <div class="form-row"> + <input type="submit" name="save_button" value="Save"/> + <input type="submit" name="cancel_button" value="Close"/> + </div> +</form> --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1584,7 +1584,7 @@ class Sample( object ): COMPLETE = 'Complete', ERROR = 'Error') def __init__(self, name=None, desc=None, request=None, form_values=None, - bar_code=None, library=None, folder=None, dataset_files=None): + bar_code=None, library=None, folder=None): self.name = name self.desc = desc self.request = request @@ -1592,27 +1592,26 @@ class Sample( object ): self.bar_code = bar_code self.library = library self.folder = folder - self.dataset_files = dataset_files def current_state(self): if self.events: return self.events[0].state return None def untransferred_dataset_files(self): count = 0 - for df in self.dataset_files: - if df['status'] == self.transfer_status.NOT_STARTED: + for df in self.datasets: + if df.status == self.transfer_status.NOT_STARTED: count = count + 1 return count def inprogress_dataset_files(self): count = 0 - for df in self.dataset_files: - if df['status'] not in [self.transfer_status.NOT_STARTED, self.transfer_status.COMPLETE]: + for df in self.datasets: + if df.status not in [self.transfer_status.NOT_STARTED, self.transfer_status.COMPLETE]: count = count + 1 return count def transferred_dataset_files(self): count = 0 - for df in self.dataset_files: - if df['status'] == self.transfer_status.COMPLETE: + for df in self.datasets: + if df.status == self.transfer_status.COMPLETE: count = count + 1 return count def dataset_size(self, filepath): @@ -1639,6 +1638,16 @@ class SampleEvent( object ): self.state = sample_state self.comment = comment +class SampleDataset( object ): + def __init__(self, sample=None, name=None, file_path=None, + status=None, error_msg=None, size=None): + self.sample = sample + self.name = name + self.file_path = file_path + self.status = status + self.error_msg = error_msg + self.size = size + class UserAddress( object ): def __init__(self, user=None, desc=None, name=None, institution=None, address=None, city=None, state=None, postal_code=None, --- a/lib/galaxy/web/controllers/requests_common.py +++ b/lib/galaxy/web/controllers/requests_common.py @@ -34,7 +34,7 @@ class RequestsCommon( BaseController ): if sample.current_state().name != state: rval[id] = { "state": sample.current_state().name, - "datasets": len(sample.dataset_files), + "datasets": len(sample.datasets), "html_state": unicode( trans.fill_template( "requests/common/sample_state.mako", sample=sample, cntrller=cntrller ), 'utf-8' ), "html_datasets": unicode( trans.fill_template( "requests/common/sample_datasets.mako", trans=trans, sample=sample, cntrller=cntrller ), 'utf-8' ) } @@ -518,7 +518,6 @@ class RequestsCommon( BaseController ): barcode=s.bar_code, library=s.library, folder=s.folder, - dataset_files=s.dataset_files, field_values=s.values.content, lib_widget=lib_widget, folder_widget=folder_widget)) @@ -530,7 +529,6 @@ class RequestsCommon( BaseController ): barcode='', library=None, folder=None, - dataset_files=[], field_values=['' for field in request.type.sample_form.fields], lib_widget=lib_widget, folder_widget=folder_widget)) @@ -792,8 +790,7 @@ class RequestsCommon( BaseController ): request, form_values, current_samples[sample_index]['barcode'], current_samples[sample_index]['library'], - current_samples[sample_index]['folder'], - dataset_files=[]) + current_samples[sample_index]['folder']) trans.sa_session.add( s ) trans.sa_session.flush() @@ -1078,12 +1075,17 @@ class RequestsCommon( BaseController ): operation='show', status='error', message="Set a data library and folder for <b>%s</b> to transfer dataset(s)." % sample.name, - id=trans.security.encode_id(sample.request.id) ) ) + id=trans.security.encode_id(sample.request.id) ) ) + if cntrller == 'requests_admin': + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='manage_datasets', + sample_id=sample.id) ) + if params.get( 'folder_path', '' ): folder_path = util.restore_text( params.get( 'folder_path', '' ) ) else: - if len(sample.dataset_files): - folder_path = os.path.dirname(sample.dataset_files[-1]['filepath'][:-1]) + if len(sample.datasets): + folder_path = os.path.dirname(sample.datasets[-1]['filepath'][:-1]) else: folder_path = util.restore_text( sample.request.type.datatx_info.get('data_dir', '') ) if folder_path and folder_path[-1] != os.sep: @@ -1094,6 +1096,6 @@ class RequestsCommon( BaseController ): message = 'The sequencer login information is incomplete. Click on the <b>Sequencer information</b> to add login details.' return trans.fill_template( '/requests/common/get_data.mako', cntrller=cntrller, sample=sample, - dataset_files=sample.dataset_files, + dataset_files=sample.datasets, message=message, status=status, files=[], folder_path=folder_path ) --- a/scripts/galaxy_messaging/server/data_transfer.py +++ b/scripts/galaxy_messaging/server/data_transfer.py @@ -70,12 +70,12 @@ class DataTransfer(object): self.config_id_secret = config_id_secret count=0 while True: - index = self.get_value_index(self.dom, 'index', count) + dataset_id = self.get_value_index(self.dom, 'dataset_id', count) file = self.get_value_index(self.dom, 'file', count) name = self.get_value_index(self.dom, 'name', count) if file: self.dataset_files.append(dict(name=name, - index=int(index), + dataset_id=int(dataset_id), file=file)) else: break @@ -149,7 +149,7 @@ class DataTransfer(object): def print_ticks(d): pass for i, df in enumerate(self.dataset_files): - self.update_status(Sample.transfer_status.TRANSFERRING, df['index']) + self.update_status(Sample.transfer_status.TRANSFERRING, df['dataset_id']) try: cmd = "scp %s@%s:'%s' '%s/%s'" % ( self.username, self.host, @@ -168,7 +168,7 @@ class DataTransfer(object): raise Exception(msg) except Exception, e: msg = traceback.format_exc() - self.update_status('Error', df['index'], msg) + self.update_status('Error', df['dataset_id'], msg) def add_to_library(self): @@ -189,29 +189,17 @@ class DataTransfer(object): log.debug(e) self.error_and_exit(str(e)) - def update_status(self, status, dataset_index='All', msg=''): + def update_status(self, status, dataset_id='All', msg=''): ''' Update the data transfer status for this dataset in the database ''' try: - log.debug('Setting status "%s" for dataset "%s" of sample "%s"' % ( status, str(dataset_index), str(self.sample_id) ) ) - df = from_json_string(self.galaxydb.get_sample_dataset_files(self.sample_id)) - if dataset_index == 'All': + log.debug('Setting status "%s" for dataset "%s" of sample "%s"' % ( status, str(dataset_id), str(self.sample_id) ) ) + if dataset_id == 'All': for dataset in self.dataset_files: - df[dataset['index']]['status'] = status - if status == 'Error': - df[dataset['index']]['error_msg'] = msg - else: - df[dataset['index']]['error_msg'] = '' - + self.galaxydb.set_sample_dataset_status(dataset['dataset_id'], status, msg) else: - df[dataset_index]['status'] = status - if status == 'Error': - df[dataset_index]['error_msg'] = msg - else: - df[dataset_index]['error_msg'] = '' - - self.galaxydb.set_sample_dataset_files(self.sample_id, to_json_string(df)) + self.galaxydb.set_sample_dataset_status(dataset_id, status, msg) log.debug('done.') except: log.error(traceback.format_exc()) @@ -229,12 +217,12 @@ class DataTransfer(object): rc = rc + node.data return rc - def get_value_index(self, dom, tag_name, index): + def get_value_index(self, dom, tag_name, dataset_id): ''' This method extracts the tag value from the xml message ''' try: - nodelist = dom.getElementsByTagName(tag_name)[index].childNodes + nodelist = dom.getElementsByTagName(tag_name)[dataset_id].childNodes except: return None rc = "" --- a/scripts/galaxy_messaging/server/galaxydb_interface.py +++ b/scripts/galaxy_messaging/server/galaxydb_interface.py @@ -28,11 +28,12 @@ class GalaxyDbInterface(object): def __init__(self, dbstr): self.dbstr = dbstr self.db_engine = create_engine(self.dbstr) - self.db_engine.echo = False + self.db_engine.echo = True self.metadata = MetaData(self.db_engine) self.session = sessionmaker(bind=self.db_engine) self.event_table = Table('sample_event', self.metadata, autoload=True ) self.sample_table = Table('sample', self.metadata, autoload=True ) + self.sample_dataset_table = Table('sample_dataset', self.metadata, autoload=True ) self.request_table = Table('request', self.metadata, autoload=True ) self.request_event_table = Table('request_event', self.metadata, autoload=True ) self.state_table = Table('sample_state', self.metadata, autoload=True ) @@ -105,7 +106,7 @@ class GalaxyDbInterface(object): create_time=datetime.utcnow(), sample_id=sample_id, sample_state_id=int(new_state_id), - comment='bar code scanner') + comment='Update by barcode scan') # if all the samples for this request are in the final state # then change the request state to 'Complete' result = select(columns=[self.sample_table.c.id], @@ -126,23 +127,22 @@ class GalaxyDbInterface(object): request_id=self.request_id, state=request_state, comment='All samples of this request have finished processing.') - - def get_sample_dataset_files(self, sample_id): - subsubquery = select(columns=[self.sample_table.c.dataset_files], - whereclause=self.sample_table.c.id==sample_id) - return subsubquery.execute().fetchall()[0][0] - - def set_sample_dataset_files(self, sample_id, value): - u = self.sample_table.update(whereclause=self.sample_table.c.id==sample_id) - u.execute(dataset_files=value) - + def set_sample_dataset_status(self, id, new_status, msg=None): + u = self.sample_dataset_table.update(whereclause=self.sample_dataset_table.c.id==int(id)) + u.execute(status=new_status) + if new_status == 'Error': + u.execute(error_msg=msg) + else: + u.execute(error_msg='') + return + if __name__ == '__main__': print '''This file should not be run directly. To start the Galaxy AMQP Listener: %sh run_galaxy_listener.sh''' - dbstr = 'postgres://postgres:postgres@localhost/galaxy_uft' + dbstr = 'postgres://postgres:postgres@localhost/g2' parser = optparse.OptionParser() parser.add_option('-n', '--name', help='name of the sample field', dest='name', \ --- /dev/null +++ b/templates/admin/requests/datasets_grid.mako @@ -0,0 +1,32 @@ + + + +<%def name="custom_javascripts()"> + <script type="text/javascript"> + $("#select-dataset-action-button").bind( "click", function(e) { + alert('afdhblvi') + $.ajax({ + url: "${h.url_for( controller='requests_admin', action='remote_file_browser' )}", + data: {id: 6}, + error: function() { alert( "Couldn't create new browser" ) }, + success: function(form_html) { + show_modal("Select file", form_html, { + "Cancel": function() { window.location = "${h.url_for( controller='requests_admin', action='list' )}"; }, + "Continue": function() { $(document).trigger("convert_dbkeys"); continue_fn(); } + }); + $("#new-title").focus(); + replace_big_select_inputs(); + } + }); + } + </script> +</%def> + +<%def name="javascripts()"> + ${h.js( "galaxy.base", "galaxy.panels", "json2", "jquery", "jquery.event.drag", "jquery.autocomplete", "jquery.mousewheel", "trackster", "ui.core", "ui.sortable" )} + ${self.custom_javascripts()} + ${parent.javascripts()} +</%def> + +<%inherit file="/grid_base.mako"/> + --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -574,7 +574,6 @@ Sample.table = Table('sample', metadata, Column( "bar_code", TrimmedString( 255 ), index=True ), Column( "library_id", Integer, ForeignKey( "library.id" ), index=True ), Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ), - Column( "dataset_files", JSONType() ), Column( "deleted", Boolean, index=True, default=False ) ) SampleState.table = Table('sample_state', metadata, @@ -593,6 +592,17 @@ SampleEvent.table = Table('sample_event' Column( "sample_state_id", Integer, ForeignKey( "sample_state.id" ), index=True ), Column( "comment", TEXT ) ) +SampleDataset.table = Table('sample_dataset', metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "sample_id", Integer, ForeignKey( "sample.id" ), index=True ), + Column( "name", TrimmedString( 255 ), nullable=False ), + Column( "file_path", TrimmedString( 255 ), nullable=False ), + Column( "status", TrimmedString( 255 ), nullable=False ), + Column( "error_msg", TEXT ), + Column( "size", TrimmedString( 255 ) ) ) + Page.table = Table( "page", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -792,6 +802,8 @@ assign_mapper( context, Sample, Sample.t properties=dict( events=relation( SampleEvent, backref="sample", order_by=desc(SampleEvent.table.c.update_time) ), + datasets=relation( SampleDataset, backref="sample", + order_by=desc(SampleDataset.table.c.update_time) ), values=relation( FormValues, primaryjoin=( Sample.table.c.form_values_id == FormValues.table.c.id ) ), request=relation( Request, @@ -865,6 +877,9 @@ assign_mapper( context, SampleEvent, Sam assign_mapper( context, SampleState, SampleState.table, properties=None ) +assign_mapper( context, SampleDataset, SampleDataset.table, + properties=None ) + assign_mapper( context, UserAddress, UserAddress.table, properties=dict( user=relation( User, --- a/templates/display_common.mako +++ b/templates/display_common.mako @@ -79,6 +79,8 @@ class_plural = "Libraries" elif a_class == model.HistoryDatasetAssociation: class_plural = "Datasets" + elif a_class == model.SampleDataset: + class_plural = "Sample Datasets" elif a_class == model.FormDefinitionCurrent: class_plural = "Forms" else: --- a/templates/requests/common/get_data.mako +++ b/templates/requests/common/get_data.mako @@ -94,8 +94,7 @@ </style> -<h2>Data transfer from Sequencer</h2> -<h3>Sample "${sample.name}" of Request "${sample.request.name}"</h3> +<h2>Datasets of Sample "${sample.name}"</h2><br/><br/> --- a/templates/requests/common/show_request.mako +++ b/templates/requests/common/show_request.mako @@ -147,30 +147,29 @@ function showContent(vThis) </div><ul class="manage-table-actions"> - - %if request.unsubmitted() and request.samples: - <li> + <li><a class="action-button" id="seqreq-${request.id}-popup" class="menubutton">Sequencing Request Actions</a></li> + <div popupmenu="seqreq-${request.id}-popup"> + %if request.unsubmitted() and request.samples: <a class="action-button" confirm="More samples cannot be added to this request once it is submitted. Click OK to submit." href="${h.url_for( controller=cntrller, action='list', operation='Submit', id=trans.security.encode_id(request.id) )}"> - <span>Submit request</span></a> - </li> - %endif - %if cntrller == 'requests_admin' and trans.user_is_admin(): - %if request.submitted(): - <li> - <a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='reject', id=trans.security.encode_id(request.id))}"> - <span>Reject request</span></a> - </li> + <span>Submit</span></a> %endif - %endif - <li> + <a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='Edit', id=trans.security.encode_id(request.id))}"> + <span>Edit</span></a><a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='events', id=trans.security.encode_id(request.id) )}"><span>History</span></a> - </li> + %if cntrller == 'requests_admin' and trans.user_is_admin(): + %if request.submitted(): + <a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='reject', id=trans.security.encode_id(request.id))}"> + <span>Reject</span></a> + <a class="action-button" href="${h.url_for( controller='requests_admin', action='get_data', show_page=True, request_id=request.id)}"> + <span>Select dataset(s) to transfer</span></a> + %endif + %endif + </div><li><a class="action-button" href="${h.url_for( controller=cntrller, action='list')}"><span>Browse requests</span></a></li> - </ul> @@ -396,12 +395,21 @@ function showContent(vThis) <td>${info['name']}</td><td>${info['barcode']}</td> %if sample.request.unsubmitted(): - <td>Unsubmitted</td> + ##<td>Unsubmitted</td> + <td> + <div id="history-name-container"> + <div id="history-name" class="tooltip editable-text" title="Click to rename history">Unsubmitted</div> + </div> + </td> %else: <td id="sampleState-${sample.id}">${render_sample_state( cntrller, sample )}</td> %endif %if info['library']: - <td><a href="${h.url_for( controller='library_common', action='browse_library', cntrller='library', id=trans.security.encode_id( info['library'].id ) )}">${info['library'].name}</a></td> + %if cntrller == 'requests': + <td><a href="${h.url_for( controller='library_common', action='browse_library', cntrller='library', id=trans.security.encode_id( info['library'].id ) )}">${info['library'].name}</a></td> + %elif cntrller == 'requests_admin': + <td><a href="${h.url_for( controller='library_common', action='browse_library', cntrller='library_admin', id=trans.security.encode_id( info['library'].id ) )}">${info['library'].name}</a></td> + %endif %else: <td></td> %endif --- a/templates/admin/requests/dataset.mako +++ b/templates/admin/requests/dataset.mako @@ -12,60 +12,44 @@ <ul class="manage-table-actions"><li><a class="action-button" href="${h.url_for( controller='requests_common', action='show_datatx_page', cntrller='requests_admin', sample_id=trans.security.encode_id(sample.id) )}"> - <span>Dataset transfer page</span></a> + <span>Browse datasets</span></a></li></ul><div class="toolForm"> - <div class="toolFormTitle">Dataset details</div> + <div class="toolFormTitle">Dataset Information</div><div class="toolFormBody"> - <form name="dataset_details" action="${h.url_for( controller='requests_admin', action='dataset_details', save_changes=True, sample_id=trans.security.encode_id(sample.id), dataset_index=dataset_index )}" method="post" > - <% - dataset = sample.dataset_files[dataset_index] - %><div class="form-row"><label>Name:</label><div style="float: left; width: 250px; margin-right: 10px;"> - %if dataset['status'] in [sample.transfer_status.IN_QUEUE, sample.transfer_status.NOT_STARTED]: - <input type="text" name="name" value="${dataset['name']}" size="60"/> - %else: - ${dataset['name']} - %endif - + ${sample_dataset.name} </div><div style="clear: both"></div></div><div class="form-row"><label>File on the Sequencer:</label><div style="float: left; width: 250px; margin-right: 10px;"> - ${dataset['filepath']} - ##<input type="text" name="filepath" value="${dataset['filepath']}" size="100" readonly/> + ${sample_dataset.file_path} </div><div style="clear: both"></div></div><div class="form-row"><label>Size:</label><div style="float: left; width: 250px; margin-right: 10px;"> - ${dataset.get('size', 'Unknown')} + ${sample_dataset.size} </div><div style="clear: both"></div></div><div class="form-row"><label>Transfer status:</label><div style="float: left; width: 250px; margin-right: 10px;"> - ${dataset['status']} + ${sample_dataset.status} <br/> - %if dataset['status'] == sample.transfer_status.ERROR: - ${dataset['error_msg']} + %if sample_dataset.status == sample.transfer_status.ERROR: + ${sample_dataset.error_msg} %endif </div><div style="clear: both"></div></div> - %if dataset['status'] in [sample.transfer_status.IN_QUEUE, sample.transfer_status.NOT_STARTED]: - <div class="form-row"> - <input type="submit" name="save" value="Save"/> - </div> - %endif - </form></div></div> --- a/lib/galaxy/web/controllers/requests_admin.py +++ b/lib/galaxy/web/controllers/requests_admin.py @@ -146,7 +146,7 @@ class RequestsGrid( grids.Grid ): # -# ---- Request Type Gridr ------------------------------------------------------ +# ---- Request Type Grid ------------------------------------------------------ # class RequestTypeGrid( grids.Grid ): # Custom column types @@ -207,6 +207,57 @@ class RequestTypeGrid( grids.Grid ): action='create_request_type' ) ) ] + +# ---- Data Transfer Grid ------------------------------------------------------ +# +class DataTransferGrid( grids.Grid ): + # Custom column types + class NameColumn( grids.TextColumn ): + def get_value(self, trans, grid, sample_dataset): + return sample_dataset.name + class SizeColumn( grids.TextColumn ): + def get_value(self, trans, grid, sample_dataset): + return sample_dataset.size + class StatusColumn( grids.TextColumn ): + def get_value(self, trans, grid, sample_dataset): + return sample_dataset.status + # Grid definition + title = "Sample Datasets" + template = "admin/requests/datasets_grid.mako" + model_class = model.SampleDataset + default_sort_key = "-create_time" + num_rows_per_page = 50 + preserve_state = True + use_paging = True + #default_filter = dict( deleted="False" ) + columns = [ + NameColumn( "Name", + #key="name", + model_class=model.SampleDataset, + link=( lambda item: dict( operation="view", id=item.id ) ), + attach_popup=True, + filterable="advanced" ), + SizeColumn( "Size", + #key='size', + model_class=model.SampleDataset, + filterable="advanced" ), + StatusColumn( "Status", + #key='status', + model_class=model.SampleDataset, + filterable="advanced" ), + ] + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ + grids.GridOperation( "Start Transfer", allow_multiple=True, condition=( lambda item: item.status in [model.Sample.transfer_status.NOT_STARTED] ) ), + grids.GridOperation( "Rename", allow_multiple=True, allow_popup=False, condition=( lambda item: item.status in [model.Sample.transfer_status.NOT_STARTED] ) ), + grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: item.status in [model.Sample.transfer_status.NOT_STARTED] ) ), + ] + def apply_query_filter( self, trans, query, **kwd ): + return query.filter_by( sample_id=kwd['sample_id'] ) # # ---- Request Controller ------------------------------------------------------ # @@ -214,6 +265,7 @@ class RequestTypeGrid( grids.Grid ): class RequestsAdmin( BaseController ): request_grid = RequestsGrid() requesttype_grid = RequestTypeGrid() + datatx_grid = DataTransferGrid() @web.expose @@ -280,11 +332,11 @@ class RequestsAdmin( BaseController ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - sample = trans.sa_session.query( self.app.model.Sample ).get( int(id) ) - datatx_info = sample.request.type.datatx_info + request = trans.sa_session.query( self.app.model.Request ).get( int(id) ) + datatx_info = request.type.datatx_info cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( datatx_info['username'], - datatx_info['host'], - folder_path ) + datatx_info['host'], + folder_path ) output = pexpect.run(cmd, events={'.ssword:*': datatx_info['password']+'\r\n', pexpect.TIMEOUT:print_ticks}, timeout=10) @@ -297,8 +349,8 @@ class RequestsAdmin( BaseController ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - sample = trans.sa_session.query( self.app.model.Sample ).get( int(id) ) - return self.__get_files(trans, sample, folder_path) + request = trans.sa_session.query( self.app.model.Request ).get( int(id) ) + return self.__get_files(trans, request.type, folder_path) def __reject_request(self, trans, **kwd): try: @@ -522,12 +574,120 @@ class RequestsAdmin( BaseController ): # # Data transfer from sequencer # + + @web.expose + @web.require_admin + def manage_datasets( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd['operation'].lower() + if not kwd.get( 'id', None ): + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='list', + status='error', + message="Invalid sample dataset ID") ) + if operation == "view": + sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(kwd['id']) ) + return trans.fill_template( '/admin/requests/dataset.mako', + sample=sample_dataset.sample, + sample_dataset=sample_dataset) - def __get_files(self, trans, sample, folder_path): + elif operation == "delete": + id_list = util.listify( kwd['id'] ) + for id in id_list: + sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id) ) + sample_id = sample_dataset.sample_id + trans.sa_session.delete( sample_dataset ) + trans.sa_session.flush() + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='manage_datasets', + sample_id=sample_id, + status='done', + message="%i dataset(s) have been removed." % len(id_list)) ) + + elif operation == "rename": + id_list = util.listify( kwd['id'] ) + sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id_list[0]) ) + return trans.fill_template( '/admin/requests/rename_datasets.mako', + sample=sample_dataset.sample, + id_list=id_list ) + elif operation == "start transfer": + id_list = util.listify( kwd['id'] ) + sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id_list[0]) ) + self.__start_datatx(trans, sample_dataset.sample, id_list) + + + # Render the grid view + try: + sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] ) + except: + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='list', + status='error', + message="Invalid sample ID" ) ) + self.datatx_grid.title = 'Datasets of Sample "%s"' % sample.name + self.datatx_grid.global_actions = [ + grids.GridAction( "Refresh", + dict( controller='requests_admin', + action='manage_datasets', + sample_id=sample.id) ), + grids.GridAction( "Select Datasets", + dict(controller='requests_admin', + action='get_data', + request_id=sample.request.id, + folder_path=sample.request.type.datatx_info['data_dir'], + sample_id=sample.id, + show_page=True)), + grids.GridAction( 'Data Library "%s"' % sample.library.name, + dict(controller='library_common', + action='browse_library', + cntrller='library_admin', + id=trans.security.encode_id( sample.library.id))), + grids.GridAction( "Browse this request", + dict( controller='requests_admin', + action='list', + operation='show', + id=trans.security.encode_id(sample.request.id)))] + return self.datatx_grid( trans, **kwd ) + + @web.expose + @web.require_admin + def rename_datasets( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + try: + sample = trans.sa_session.query( trans.app.model.Sample ).get( trans.security.decode_id(kwd['sample_id'])) + except: + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='list', + status='error', + message="Invalid sample ID" ) ) + if params.get( 'save_button', False ): + id_list = util.listify( kwd['id_list'] ) + for id in id_list: + sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id) ) + prepend = util.restore_text( params.get( 'prepend_%i' % sample_dataset.id, '' ) ) + name = util.restore_text( params.get( 'name_%i' % sample_dataset.id, sample_dataset.name ) ) + if prepend == 'None': + sample_dataset.name = name + else: + sample_dataset.name = prepend+'_'+name + trans.sa_session.add( sample_dataset ) + trans.sa_session.flush() + return trans.fill_template( '/admin/requests/rename_datasets.mako', + sample=sample, id_list=id_list, + message='Changes saved successfully.', + status='done' ) + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='manage_datasets', + sample_id=sample.id) ) + + + def __get_files(self, trans, request_type, folder_path): ''' This method retrieves the filenames to be transfer from the remote host. ''' - datatx_info = sample.request.type.datatx_info + datatx_info = request_type.datatx_info if not datatx_info['host'] or not datatx_info['username'] or not datatx_info['password']: message = "Error in sequencer login information." return trans.response.send_redirect( web.url_for( controller='requests_common', @@ -555,123 +715,152 @@ class RequestsAdmin( BaseController ): return output.splitlines() - def __get_files_in_dir(self, trans, sample, folder_path): - tmpfiles = self.__get_files(trans, sample, folder_path) - for tf in tmpfiles: - if tf[-1] == os.sep: - self.__get_files_in_dir(trans, sample, os.path.join(folder_path, tf)) +# def __get_files_in_dir(self, trans, sample, folder_path): +# tmpfiles = self.__get_files(trans, sample, folder_path) +# for tf in tmpfiles: +# if tf[-1] == os.sep: +# self.__get_files_in_dir(trans, sample, os.path.join(folder_path, tf)) +# else: +# sample.dataset_files.append([os.path.join(folder_path, tf), +# sample.transfer_status.NOT_STARTED]) +# trans.sa_session.add( sample ) +# trans.sa_session.flush() +# return + + + def __samples_selectbox(self, trans, request, sample_id=None): + samples_selectbox = SelectField('sample_id') + for i, s in enumerate(request.samples): + if str(s.id) == sample_id: + samples_selectbox.add_option(s.name, s.id, selected=True) else: - sample.dataset_files.append([os.path.join(folder_path, tf), - sample.transfer_status.NOT_STARTED]) - trans.sa_session.add( sample ) - trans.sa_session.flush() - return + samples_selectbox.add_option(s.name, s.id) + return samples_selectbox + @web.expose @web.require_admin def get_data(self, trans, **kwd): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) try: - sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] ) + request = trans.sa_session.query( trans.app.model.Request ).get( kwd['request_id'] ) except: return trans.response.send_redirect( web.url_for( controller='requests_admin', action='list', status='error', - message="Invalid sample ID" ) ) - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message="Invalid request ID" ) ) + files_list = util.listify( params.get( 'files_list', '' ) ) folder_path = util.restore_text( params.get( 'folder_path', - sample.request.type.datatx_info['data_dir'] ) ) - files_list = util.listify( params.get( 'files_list', '' ) ) - if params.get( 'start_transfer_button', False ) == 'True': - return self.__start_datatx(trans, sample) + request.type.datatx_info['data_dir'] ) ) + sbox = self.__samples_selectbox(trans, request, kwd.get('sample_id', None)) if not folder_path: - return trans.fill_template( '/requests/common/get_data.mako', - cntrller='requests_admin', - sample=sample, files=[], - dataset_files=sample.dataset_files, + return trans.fill_template( '/admin/requests/get_data.mako', + cntrller='requests_admin', request=request, + samples_selectbox=sbox, files=[], folder_path=folder_path ) if folder_path[-1] != os.sep: folder_path = folder_path+os.sep - if params.get( 'browse_button', False ): + if params.get( 'show_page', False ): + if kwd.get('sample_id', None): + sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] ) + if sample.datasets: + folder_path = os.path.dirname(sample.datasets[-1].file_path) + return trans.fill_template( '/admin/requests/get_data.mako', + cntrller='requests_admin', request=request, + samples_selectbox=sbox, files=[], + folder_path=folder_path, + status=status, message=message ) + elif params.get( 'browse_button', False ): # get the filenames from the remote host - files = self.__get_files(trans, sample, folder_path) + files = self.__get_files(trans, request.type, folder_path) if folder_path[-1] != os.sep: folder_path += os.sep - return trans.fill_template( '/requests/common/get_data.mako', - cntrller='requests_admin', - sample=sample, files=files, - dataset_files=sample.dataset_files, - folder_path=folder_path ) + return trans.fill_template( '/admin/requests/get_data.mako', + cntrller='requests_admin', request=request, + samples_selectbox=sbox, files=files, + folder_path=folder_path, + status=status, message=message ) elif params.get( 'folder_up', False ): if folder_path[-1] == os.sep: folder_path = os.path.dirname(folder_path[:-1]) # get the filenames from the remote host - files = self.__get_files(trans, sample, folder_path) + files = self.__get_files(trans, request.type, folder_path) if folder_path[-1] != os.sep: folder_path += os.sep - return trans.fill_template( '/requests/common/get_data.mako', - cntrller='requests_admin', - sample=sample, files=files, - dataset_files=sample.dataset_files, - folder_path=folder_path ) + return trans.fill_template( '/admin/requests/get_data.mako', + cntrller='requests_admin',request=request, + samples_selectbox=sbox, files=files, + folder_path=folder_path, + status=status, message=message ) elif params.get( 'open_folder', False ): if len(files_list) == 1: folder_path = os.path.join(folder_path, files_list[0]) # get the filenames from the remote host - files = self.__get_files(trans, sample, folder_path) + files = self.__get_files(trans, request.type, folder_path) if folder_path[-1] != os.sep: folder_path += os.sep - return trans.fill_template( '/requests/common/get_data.mako', - cntrller='requests_admin', - sample=sample, files=files, - dataset_files=sample.dataset_files, - folder_path=folder_path ) - elif params.get( 'remove_dataset_button', False ): - # get the filenames from the remote host - files = self.__get_files(trans, sample, folder_path) - dataset_index = int(params.get( 'dataset_index', 0 )) - del sample.dataset_files[dataset_index] - trans.sa_session.add( sample ) - trans.sa_session.flush() - return trans.fill_template( '/requests/common/get_data.mako', - cntrller='requests_admin', - sample=sample, files=files, - dataset_files=sample.dataset_files, - folder_path=folder_path) - elif params.get( 'select_files_button', False ): - folder_files = [] - if len(files_list): - for f in files_list: - filepath = os.path.join(folder_path, f) - if f[-1] == os.sep: - # the selected item is a folder so transfer all the - # folder contents - # FIXME - #self.__get_files_in_dir(trans, sample, filepath) - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='get_data', - sample_id=sample.id, - folder_path=folder_path, - open_folder=True)) - else: - sample.dataset_files.append(dict(filepath=filepath, - status=sample.transfer_status.NOT_STARTED, - name=self.__dataset_name(sample, filepath.split('/')[-1]), - error_msg='', - size=sample.dataset_size(filepath))) - trans.sa_session.add( sample ) - trans.sa_session.flush() + return trans.fill_template( '/admin/requests/get_data.mako', + cntrller='requests_admin', request=request, + samples_selectbox=sbox, files=files, + folder_path=folder_path, + status=status, message=message ) + elif params.get( 'select_show_datasets_button', False ): + sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] ) + retval = self.__save_sample_datasets(trans, sample, files_list, folder_path) + if retval: message='The dataset(s) %s have been selected for sample <b>%s</b>' %(str(retval)[1:-1].replace("'", ""), sample.name) + else: message = None + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='manage_datasets', + sample_id=sample.id, + status='done', + message=message) ) + elif params.get( 'select_more_button', False ): + sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] ) + retval = self.__save_sample_datasets(trans, sample, files_list, folder_path) + if retval: message='The dataset(s) %s have been selected for sample <b>%s</b>' %(str(retval)[1:-1].replace("'", ""), sample.name) + else: message = None return trans.response.send_redirect( web.url_for( controller='requests_admin', action='get_data', + request_id=sample.request.id, + folder_path=folder_path, sample_id=sample.id, - folder_path=folder_path, - open_folder=True)) - - return trans.response.send_redirect( web.url_for( controller='requests_common', - cntrller='requests_admin' , - action='show_datatx_page', - sample_id=trans.security.encode_id(sample.id), - folder_path=folder_path)) + open_folder=True, + status='done', + message=message)) + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='get_data', + request_id=sample.request.id, + folder_path=folder_path, + show_page=True)) + + def __save_sample_datasets(self, trans, sample, files_list, folder_path): + files = [] + if len(files_list): + for f in files_list: + filepath = os.path.join(folder_path, f) + if f[-1] == os.sep: + # the selected item is a folder so transfer all the + # folder contents + # FIXME + #self.__get_files_in_dir(trans, sample, filepath) + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='get_data', + request=sample.request, + folder_path=folder_path, + open_folder=True)) + else: + sample_dataset = trans.app.model.SampleDataset( sample=sample, + file_path=filepath, + status=sample.transfer_status.NOT_STARTED, + name=self.__dataset_name(sample, filepath.split('/')[-1]), + error_msg='', + size=sample.dataset_size(filepath)) + trans.sa_session.add( sample_dataset ) + trans.sa_session.flush() + files.append(str(sample_dataset.name)) + return files + def __dataset_name(self, sample, filepath): name = filepath.split('/')[-1] @@ -735,7 +924,7 @@ class RequestsAdmin( BaseController ): trans.sa_session.flush() return datatx_user - def __send_message(self, trans, datatx_info, sample): + def __send_message(self, trans, datatx_info, sample, id_list): ''' This method creates the xml message and sends it to the rabbitmq server ''' @@ -752,20 +941,20 @@ class RequestsAdmin( BaseController ): </data_transfer>''' dataset_xml = \ '''<dataset> - <index>%(INDEX)s</index> + <dataset_id>%(ID)s</dataset_id><name>%(NAME)s</name><file>%(FILE)s</file></dataset>''' datasets = '' - for index, dataset in enumerate(sample.dataset_files): - if dataset['status'] == sample.transfer_status.NOT_STARTED: - datasets = datasets + dataset_xml % dict(INDEX=str(index), - NAME=dataset['name'], - FILE=dataset['filepath']) - sample.dataset_files[index]['status'] = sample.transfer_status.IN_QUEUE - - trans.sa_session.add( sample ) - trans.sa_session.flush() + for id in id_list: + sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id) ) + if sample_dataset.status == sample.transfer_status.NOT_STARTED: + datasets = datasets + dataset_xml % dict(ID=str(sample_dataset.id), + NAME=sample_dataset.name, + FILE=sample_dataset.file_path) + sample_dataset.status = sample.transfer_status.IN_QUEUE + trans.sa_session.add( sample_dataset ) + trans.sa_session.flush() data = xml % dict(DATA_HOST=datatx_info['host'], DATA_USER=datatx_info['username'], DATA_PASSWORD=datatx_info['password'], @@ -790,7 +979,7 @@ class RequestsAdmin( BaseController ): chan.close() conn.close() - def __start_datatx(self, trans, sample): + def __start_datatx(self, trans, sample, id_list): # data transfer user datatx_user = self.__setup_datatx_user(trans, sample.library, sample.folder) # validate sequecer information @@ -799,46 +988,19 @@ class RequestsAdmin( BaseController ): not datatx_info['username'] or \ not datatx_info['password']: message = "Error in sequencer login information." - return trans.response.send_redirect( web.url_for( controller='requests_common', - cntrller='requests_admin' , - action='show_datatx_page', - sample_id=trans.security.encode_id(sample.id), + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='manage_datasets', + sample_id=sample.id, status='error', - message=message)) - self.__send_message(trans, datatx_info, sample) - return trans.response.send_redirect( web.url_for( controller='requests_common', - cntrller='requests_admin' , - action='show_datatx_page', - sample_id=trans.security.encode_id(sample.id), - folder_path=datatx_info['data_dir'])) - @web.expose - @web.require_admin - def dataset_details( self, trans, **kwd ): - try: - sample = trans.sa_session.query( trans.app.model.Sample ).get( trans.security.decode_id(kwd['sample_id']) ) - except: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='list', - status='error', - message="Invalid sample ID" ) ) - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - dataset_index = int( params.get( 'dataset_index', '' ) ) - if params.get('save', '') == 'Save': - sample.dataset_files[dataset_index]['name'] = util.restore_text( params.get( 'name', - sample.dataset_files[dataset_index]['name'] ) ) - trans.sa_session.add( sample ) - trans.sa_session.flush() - status = 'done' - message = 'Saved the changes made to the dataset.' - return trans.fill_template( '/admin/requests/dataset.mako', - sample=sample, - dataset_index=dataset_index, - message=message, - status=status) + message=message) ) + self.__send_message(trans, datatx_info, sample, id_list) + message="%i dataset(s) have been queued for transfer from the sequencer. Click on <b>Refresh</b> button above to get the latest transfer status." % len(id_list) + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='manage_datasets', + sample_id=sample.id, + status='done', + message=message) ) -# ## #### Request Type Stuff ################################################### ##