details: http://www.bx.psu.edu/hg/galaxy/rev/7fd4f748b0ca changeset: 2375:7fd4f748b0ca user: Dan Blankenberg <dan(a)bx.psu.edu&gt; date: Mon Apr 27 16:30:08 2009 -0400 description: Remove direct references of the model from the the cleanup_datasets_fix migration script. 1 file(s) affected in this change: lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py diffs (709 lines): diff -r d0c905db68db -r 7fd4f748b0ca lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py Mon Apr 27 16:30:08 2009 -0400 @@ -0,0 +1,705 @@ +import sys, logging, os, time, datetime, errno + +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +from migrate import migrate_engine +from sqlalchemy import and_ + +from sqlalchemy import * +now = datetime.datetime.utcnow +from sqlalchemy.orm import * + +from galaxy.model.orm.ext.assignmapper import assign_mapper + +from galaxy.model.custom_types import * + +from galaxy.util.bunch import Bunch + + +metadata = MetaData( migrate_engine ) +context = scoped_session( sessionmaker( autoflush=False, transactional=False ) ) + + +## classes +def get_permitted_actions( **kwds ): + return Bunch() + +def directory_hash_id( id ): + s = str( id ) + l = len( s ) + # Shortcut -- ids 0-999 go under ../000/ + if l < 4: + return [ "000" ] + # Pad with zeros until a multiple of three + padded = ( ( 3 - len( s ) % 3 ) * "0" ) + s + # Drop the last three digits -- 1000 files per directory + padded = padded[:-3] + # Break into chunks of three + return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ] + + +class Dataset( object ): + states = Bunch( NEW = 'new', + UPLOAD = 'upload', + QUEUED = 'queued', + RUNNING = 'running', + OK = 'ok', + EMPTY = 'empty', + ERROR = 'error', + DISCARDED = 'discarded' ) + permitted_actions = get_permitted_actions( filter='DATASET' ) + file_path = "/tmp/" + engine = None + def __init__( self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True ): + self.id = id + self.state = state + self.deleted = False + self.purged = False + self.purgable = purgable + self.external_filename = external_filename + self._extra_files_path = extra_files_path + self.file_size = file_size + def get_file_name( self ): + if not self.external_filename: + assert self.id is not None, "ID must be set before filename used (commit the object)" + # First try filename directly under file_path + filename = os.path.join( self.file_path, "dataset_%d.dat" % self.id ) + # Only use that filename if it already exists (backward compatibility), + # otherwise construct hashed path + if not os.path.exists( filename ): + dir = os.path.join( self.file_path, *directory_hash_id( self.id ) ) + # Create directory if it does not exist + try: + os.makedirs( dir ) + except OSError, e: + # File Exists is okay, otherwise reraise + if e.errno != errno.EEXIST: + raise + # Return filename inside hashed directory + return os.path.abspath( os.path.join( dir, "dataset_%d.dat" % self.id ) ) + else: + filename = self.external_filename + # Make filename absolute + return os.path.abspath( filename ) + def set_file_name ( self, filename ): + if not filename: + self.external_filename = None + else: + self.external_filename = filename + file_name = property( get_file_name, set_file_name ) + @property + def extra_files_path( self ): + if self._extra_files_path: + path = self._extra_files_path + else: + path = os.path.join( self.file_path, "dataset_%d_files" % self.id ) + #only use path directly under self.file_path if it exists + if not os.path.exists( path ): + path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id ) + # Make path absolute + return os.path.abspath( path ) + def get_size( self ): + """Returns the size of the data on disk""" + if self.file_size: + return self.file_size + else: + try: + return os.path.getsize( self.file_name ) + except OSError: + return 0 + def set_size( self ): + """Returns the size of the data on disk""" + try: + if not self.file_size: + self.file_size = os.path.getsize( self.file_name ) + except OSError: + self.file_size = 0 + def has_data( self ): + """Detects whether there is any data""" + return self.get_size() > 0 + def mark_deleted( self, include_children=True ): + self.deleted = True + # FIXME: sqlalchemy will replace this + def _delete(self): + """Remove the file that corresponds to this data""" + try: + os.remove(self.data.file_name) + except OSError, e: + log.critical('%s delete error %s' % (self.__class__.__name__, e)) + +class DatasetInstance( object ): + """A base class for all 'dataset instances', HDAs, LDAs, etc""" + states = Dataset.states + permitted_actions = Dataset.permitted_actions + def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, + dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, + parent_id=None, validation_errors=None, visible=True, create_dataset = False ): + self.name = name or "Unnamed dataset" + self.id = id + self.info = info + self.blurb = blurb + self.peek = peek + self.extension = extension + self.designation = designation + self.metadata = metadata or dict() + if dbkey: #dbkey is stored in metadata, only set if non-zero, or else we could clobber one supplied by input 'metadata' + self.dbkey = dbkey + self.deleted = deleted + self.visible = visible + # Relationships + if not dataset and create_dataset: + dataset = Dataset( state=Dataset.states.NEW ) + dataset.flush() + self.dataset = dataset + self.parent_id = parent_id + self.validation_errors = validation_errors + @property + def ext( self ): + return self.extension + def get_dataset_state( self ): + return self.dataset.state + def set_dataset_state ( self, state ): + self.dataset.state = state + self.dataset.flush() #flush here, because hda.flush() won't flush the Dataset object + state = property( get_dataset_state, set_dataset_state ) + def get_file_name( self ): + return self.dataset.get_file_name() + def set_file_name (self, filename): + return self.dataset.set_file_name( filename ) + file_name = property( get_file_name, set_file_name ) + @property + def extra_files_path( self ): + return self.dataset.extra_files_path + @property + def datatype( self ): + return datatypes_registry.get_datatype_by_extension( self.extension ) + def get_metadata( self ): + if not hasattr( self, '_metadata_collection' ) or self._metadata_collection.parent != self: #using weakref to store parent (to prevent circ ref), does a context.clear() cause parent to be invalidated, while still copying over this non-database attribute? + self._metadata_collection = MetadataCollection( self ) + return self._metadata_collection + def set_metadata( self, bunch ): + # Needs to accept a MetadataCollection, a bunch, or a dict + self._metadata = self.metadata.make_dict_copy( bunch ) + metadata = property( get_metadata, set_metadata ) + # This provide backwards compatibility with using the old dbkey + # field in the database. That field now maps to "old_dbkey" (see mapping.py). + def get_dbkey( self ): + dbkey = self.metadata.dbkey + if not isinstance(dbkey, list): dbkey = [dbkey] + if dbkey in [[None], []]: return "?" + return dbkey[0] + def set_dbkey( self, value ): + if "dbkey" in self.datatype.metadata_spec: + if not isinstance(value, list): + self.metadata.dbkey = [value] + else: + self.metadata.dbkey = value + dbkey = property( get_dbkey, set_dbkey ) + def change_datatype( self, new_ext ): + self.clear_associated_files() + datatypes_registry.change_datatype( self, new_ext ) + def get_size( self ): + """Returns the size of the data on disk""" + return self.dataset.get_size() + def set_size( self ): + """Returns the size of the data on disk""" + return self.dataset.set_size() + def has_data( self ): + """Detects whether there is any data""" + return self.dataset.has_data() + def get_raw_data( self ): + """Returns the full data. To stream it open the file_name and read/write as needed""" + return self.datatype.get_raw_data( self ) + def write_from_stream( self, stream ): + """Writes data from a stream""" + self.datatype.write_from_stream(self, stream) + def set_raw_data( self, data ): + """Saves the data on the disc""" + self.datatype.set_raw_data(self, data) + def get_mime( self ): + """Returns the mime type of the data""" + return datatypes_registry.get_mimetype_by_extension( self.extension.lower() ) + def set_peek( self ): + return self.datatype.set_peek( self ) + def set_multi_byte_peek( self ): + return self.datatype.set_multi_byte_peek( self ) + def init_meta( self, copy_from=None ): + return self.datatype.init_meta( self, copy_from=copy_from ) + def set_meta( self, **kwd ): + self.clear_associated_files( metadata_safe = True ) + return self.datatype.set_meta( self, **kwd ) + def set_readonly_meta( self, **kwd ): + return self.datatype.set_readonly_meta( self, **kwd ) + def missing_meta( self, **kwd ): + return self.datatype.missing_meta( self, **kwd ) + def as_display_type( self, type, **kwd ): + return self.datatype.as_display_type( self, type, **kwd ) + def display_peek( self ): + return self.datatype.display_peek( self ) + def display_name( self ): + return self.datatype.display_name( self ) + def display_info( self ): + return self.datatype.display_info( self ) + def get_converted_files_by_type( self, file_type ): + valid = [] + for assoc in self.implicitly_converted_datasets: + if not assoc.deleted and assoc.type == file_type: + valid.append( assoc.dataset ) + return valid + def clear_associated_files( self, metadata_safe = False, purge = False ): + raise 'Unimplemented' + def get_child_by_designation(self, designation): + for child in self.children: + if child.designation == designation: + return child + return None + def get_converter_types(self): + return self.datatype.get_converter_types( self, datatypes_registry) + def find_conversion_destination( self, accepted_formats, **kwd ): + """Returns ( target_ext, exisiting converted dataset )""" + return self.datatype.find_conversion_destination( self, accepted_formats, datatypes_registry, **kwd ) + def add_validation_error( self, validation_error ): + self.validation_errors.append( validation_error ) + def extend_validation_errors( self, validation_errors ): + self.validation_errors.extend(validation_errors) + def mark_deleted( self, include_children=True ): + self.deleted = True + if include_children: + for child in self.children: + child.mark_deleted() + def mark_undeleted( self, include_children=True ): + self.deleted = False + if include_children: + for child in self.children: + child.mark_undeleted() + def undeletable( self ): + if self.purged: + return False + return True + @property + def source_library_dataset( self ): + def get_source( dataset ): + if isinstance( dataset, LibraryDatasetDatasetAssociation ): + if dataset.library_dataset: + return ( dataset, dataset.library_dataset ) + if dataset.copied_from_library_dataset_dataset_association: + source = get_source( dataset.copied_from_library_dataset_dataset_association ) + if source: + return source + if dataset.copied_from_history_dataset_association: + source = get_source( dataset.copied_from_history_dataset_association ) + if source: + return source + return ( None, None ) + return get_source( self ) + + +class HistoryDatasetAssociation( DatasetInstance ): + def __init__( self, + hid = None, + history = None, + copied_from_history_dataset_association = None, + copied_from_library_dataset_dataset_association = None, + **kwd ): + DatasetInstance.__init__( self, **kwd ) + self.hid = hid + # Relationships + self.history = history + self.copied_from_history_dataset_association = copied_from_history_dataset_association + self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association + def copy( self, copy_children = False, parent_id = None, target_history = None ): + hda = HistoryDatasetAssociation( hid=self.hid, + name=self.name, + info=self.info, + blurb=self.blurb, + peek=self.peek, + extension=self.extension, + dbkey=self.dbkey, + dataset = self.dataset, + visible=self.visible, + deleted=self.deleted, + parent_id=parent_id, + copied_from_history_dataset_association=self, + history = target_history ) + hda.flush() + hda.set_size() + # Need to set after flushed, as MetadataFiles require dataset.id + hda.metadata = self.metadata + if copy_children: + for child in self.children: + child_copy = child.copy( copy_children = copy_children, parent_id = hda.id ) + if not self.datatype.copy_safe_peek: + # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs + hda.set_peek() + hda.flush() + return hda + def to_library_dataset_dataset_association( self, target_folder, replace_dataset=None, parent_id=None ): + if replace_dataset: + # The replace_dataset param ( when not None ) refers to a LibraryDataset that is being replaced with a new version. + library_dataset = replace_dataset + else: + # If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new + # LibraryDataset, and the current user's DefaultUserPermissions will be applied to the associated Dataset. + library_dataset = LibraryDataset( folder=target_folder, name=self.name, info=self.info ) + library_dataset.flush() + ldda = LibraryDatasetDatasetAssociation( name=self.name, + info=self.info, + blurb=self.blurb, + peek=self.peek, + extension=self.extension, + dbkey=self.dbkey, + dataset=self.dataset, + library_dataset=library_dataset, + visible=self.visible, + deleted=self.deleted, + parent_id=parent_id, + copied_from_history_dataset_association=self, + user=self.history.user ) + ldda.flush() + # Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset + # Must set metadata after ldda flushed, as MetadataFiles require ldda.id + ldda.metadata = self.metadata + if not replace_dataset: + target_folder.add_library_dataset( library_dataset, genome_build=ldda.dbkey ) + target_folder.flush() + library_dataset.library_dataset_dataset_association_id = ldda.id + library_dataset.flush() + for child in self.children: + child_copy = child.to_library_dataset_dataset_association( target_folder=target_folder, replace_dataset=replace_dataset, parent_id=ldda.id ) + if not self.datatype.copy_safe_peek: + # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs + ldda.set_peek() + ldda.flush() + return ldda + def clear_associated_files( self, metadata_safe = False, purge = False ): + # metadata_safe = True means to only clear when assoc.metadata_safe == False + for assoc in self.implicitly_converted_datasets: + if not metadata_safe or not assoc.metadata_safe: + assoc.clear( purge = purge ) + + + +class LibraryDatasetDatasetAssociation( DatasetInstance ): + def __init__( self, + copied_from_history_dataset_association=None, + copied_from_library_dataset_dataset_association=None, + library_dataset=None, + user=None, + **kwd ): + DatasetInstance.__init__( self, **kwd ) + self.copied_from_history_dataset_association = copied_from_history_dataset_association + self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association + self.library_dataset = library_dataset + self.user = user + def to_history_dataset_association( self, target_history, parent_id=None ): + hid = target_history._next_hid() + hda = HistoryDatasetAssociation( name=self.name, + info=self.info, + blurb=self.blurb, + peek=self.peek, + extension=self.extension, + dbkey=self.dbkey, + dataset=self.dataset, + visible=self.visible, + deleted=self.deleted, + parent_id=parent_id, + copied_from_library_dataset_dataset_association=self, + history=target_history, + hid=hid ) + hda.flush() + hda.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id + for child in self.children: + child_copy = child.to_history_dataset_association( target_history=target_history, parent_id=hda.id ) + if not self.datatype.copy_safe_peek: + hda.set_peek() #in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs + hda.flush() + return hda + def copy( self, copy_children = False, parent_id = None, target_folder = None ): + ldda = LibraryDatasetDatasetAssociation( name=self.name, + info=self.info, + blurb=self.blurb, + peek=self.peek, + extension=self.extension, + dbkey=self.dbkey, + dataset=self.dataset, + visible=self.visible, + deleted=self.deleted, + parent_id=parent_id, + copied_from_library_dataset_dataset_association=self, + folder=target_folder ) + ldda.flush() + # Need to set after flushed, as MetadataFiles require dataset.id + ldda.metadata = self.metadata + if copy_children: + for child in self.children: + child_copy = child.copy( copy_children = copy_children, parent_id = ldda.id ) + if not self.datatype.copy_safe_peek: + # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs + ldda.set_peek() + ldda.flush() + return ldda + def clear_associated_files( self, metadata_safe = False, purge = False ): + return + def get_library_item_info_templates( self, template_list=[], restrict=False ): + # If restrict is True, we'll return only those templates directly associated with this LibraryDatasetDatasetAssociation + if self.library_dataset_dataset_info_template_associations: + template_list.extend( [ lddita.library_item_info_template for lddita in self.library_dataset_dataset_info_template_associations if lddita.library_item_info_template not in template_list ] ) + self.library_dataset.get_library_item_info_templates( template_list, restrict ) + return template_list + + + +class LibraryDataset( object ): + # This class acts as a proxy to the currently selected LDDA + def __init__( self, folder=None, order_id=None, name=None, info=None, library_dataset_dataset_association=None, **kwd ): + self.folder = folder + self.order_id = order_id + self.name = name + self.info = info + self.library_dataset_dataset_association = library_dataset_dataset_association + def set_library_dataset_dataset_association( self, ldda ): + self.library_dataset_dataset_association = ldda + ldda.library_dataset = self + ldda.flush() + self.flush() + def get_info( self ): + if self.library_dataset_dataset_association: + return self.library_dataset_dataset_association.info + elif self._info: + return self._info + else: + return 'no info' + def set_info( self, info ): + self._info = info + info = property( get_info, set_info ) + def get_name( self ): + if self.library_dataset_dataset_association: + return self.library_dataset_dataset_association.name + elif self._name: + return self._name + else: + return 'Unnamed dataset' + def set_name( self, name ): + self._name = name + name = property( get_name, set_name ) + def display_name( self ): + self.library_dataset_dataset_association.display_name() + def get_purged( self ): + return self.library_dataset_dataset_association.dataset.purged + def set_purged( self, purged ): + if purged: + raise Exception( "Not implemented" ) + if not purged and self.purged: + raise Exception( "Cannot unpurge once purged" ) + purged = property( get_purged, set_purged ) + def get_library_item_info_templates( self, template_list=[], restrict=False ): + # If restrict is True, we'll return only those templates directly associated with this LibraryDataset + if self.library_dataset_info_template_associations: + template_list.extend( [ ldita.library_item_info_template for ldita in self.library_dataset_info_template_associations if ldita.library_item_info_template not in template_list ] ) + if restrict not in [ 'True', True ]: + self.folder.get_library_item_info_templates( template_list, restrict ) + return template_list + +##tables + + +Dataset.table = Table( "dataset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "state", TrimmedString( 64 ) ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "purged", Boolean, index=True, default=False ), + Column( "purgable", Boolean, default=True ), + Column( "external_filename" , TEXT ), + Column( "_extra_files_path", TEXT ), + Column( 'file_size', Numeric( 15, 0 ) ) ) + + + +HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "copied_from_history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ), + Column( "copied_from_library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True ), + Column( "hid", Integer ), + Column( "name", TrimmedString( 255 ) ), + Column( "info", TrimmedString( 255 ) ), + Column( "blurb", TrimmedString( 255 ) ), + Column( "peek" , TEXT ), + Column( "extension", TrimmedString( 64 ) ), + Column( "metadata", MetadataType(), key="_metadata" ), + Column( "parent_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ), + Column( "designation", TrimmedString( 255 ) ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "visible", Boolean ) ) + + +LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "library_dataset_id", Integer, ForeignKey( "library_dataset.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "copied_from_history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id", use_alter=True, name='history_dataset_association_dataset_id_fkey' ), nullable=True ), + Column( "copied_from_library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name='library_dataset_dataset_association_id_fkey' ), nullable=True ), + Column( "name", TrimmedString( 255 ) ), + Column( "info", TrimmedString( 255 ) ), + Column( "blurb", TrimmedString( 255 ) ), + Column( "peek" , TEXT ), + Column( "extension", TrimmedString( 64 ) ), + Column( "metadata", MetadataType(), key="_metadata" ), + Column( "parent_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True ), + Column( "designation", TrimmedString( 255 ) ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "visible", Boolean ), + Column( "message", TrimmedString( 255 ) ) ) + +LibraryDataset.table = Table( "library_dataset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ),#current version of dataset, if null, there is not a current version selected + Column( "order_id", Integer ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "name", TrimmedString( 255 ), key="_name" ), #when not None/null this will supercede display in library (but not when imported into user's history?) + Column( "info", TrimmedString( 255 ), key="_info" ), #when not None/null this will supercede display in library (but not when imported into user's history?) + Column( "deleted", Boolean, index=True, default=False ) ) + + + +##mappers + + +assign_mapper( context, Dataset, Dataset.table, + properties=dict( + history_associations=relation( + HistoryDatasetAssociation, + primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) ), + active_history_associations=relation( + HistoryDatasetAssociation, + primaryjoin=( ( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) & ( HistoryDatasetAssociation.table.c.deleted == False ) ) ), + library_associations=relation( + LibraryDatasetDatasetAssociation, + primaryjoin=( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) ), + active_library_associations=relation( + LibraryDatasetDatasetAssociation, + primaryjoin=( ( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( LibraryDatasetDatasetAssociation.table.c.deleted == False ) ) ) + ) ) + + +assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.table, + properties=dict( + dataset=relation( + Dataset, + primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ), lazy=False ), + # .history defined in History mapper + copied_to_history_dataset_associations=relation( + HistoryDatasetAssociation, + primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ), + backref=backref( "copied_from_history_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False ) ), + copied_to_library_dataset_dataset_associations=relation( + LibraryDatasetDatasetAssociation, + primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), + backref=backref( "copied_from_history_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id], uselist=False ) ), + children=relation( + HistoryDatasetAssociation, + primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), + backref=backref( "parent", primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False ) ), + visible_children=relation( + HistoryDatasetAssociation, + primaryjoin=( ( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ) & ( HistoryDatasetAssociation.table.c.visible == True ) ) ) + ) ) + +assign_mapper( context, LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table, + properties=dict( + dataset=relation( Dataset ), + library_dataset = relation( LibraryDataset, + primaryjoin=( LibraryDatasetDatasetAssociation.table.c.library_dataset_id == LibraryDataset.table.c.id ) ), + copied_to_library_dataset_dataset_associations=relation( + LibraryDatasetDatasetAssociation, + primaryjoin=( LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), + backref=backref( "copied_from_library_dataset_dataset_association", primaryjoin=( LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id] ) ), + copied_to_history_dataset_associations=relation( + HistoryDatasetAssociation, + primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), + backref=backref( "copied_from_library_dataset_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id], uselist=False ) ), + children=relation( + LibraryDatasetDatasetAssociation, + primaryjoin=( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ), + backref=backref( "parent", primaryjoin=( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id] ) ), + visible_children=relation( + LibraryDatasetDatasetAssociation, + primaryjoin=( ( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ) & ( LibraryDatasetDatasetAssociation.table.c.visible == True ) ) ) + ) ) + +assign_mapper( context, LibraryDataset, LibraryDataset.table, + properties=dict( + library_dataset_dataset_association=relation( LibraryDatasetDatasetAssociation, primaryjoin=( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ), + expired_datasets = relation( LibraryDatasetDatasetAssociation, foreign_keys=[LibraryDataset.table.c.id,LibraryDataset.table.c.library_dataset_dataset_association_id ], primaryjoin=( ( LibraryDataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.library_dataset_id ) & ( not_( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ) ), viewonly=True, uselist=True ) + ) ) + + +def __guess_dataset_by_filename( filename ): + """Return a guessed dataset by filename""" + try: + fields = os.path.split( filename ) + if fields: + if fields[-1].startswith( 'dataset_' ) and fields[-1].endswith( '.dat' ): #dataset_%d.dat + return Dataset.get( int( fields[-1][ len( 'dataset_' ): -len( '.dat' ) ] ) ) + except: + pass #some parsing error, we can't guess Dataset + return None + +def upgrade(): + log.debug( "Fixing a discrepancy concerning deleted shared history items." ) + affected_items = 0 + start_time = time.time() + for dataset in Dataset.filter( and_( Dataset.c.deleted == True, Dataset.c.purged == False ) ).all(): + for dataset_instance in dataset.history_associations + dataset.library_associations: + if not dataset_instance.deleted: + dataset.deleted = False + if dataset.file_size in [ None, 0 ]: + dataset.set_size() #Restore filesize + affected_items += 1 + break + context.flush() + log.debug( "%i items affected, and restored." % ( affected_items ) ) + log.debug( "Time elapsed: %s" % ( time.time() - start_time ) ) + + #fix share before hda + log.debug( "Fixing a discrepancy concerning cleaning up deleted history items shared before HDAs." ) + dataset_by_filename = {} + changed_associations = 0 + start_time = time.time() + for dataset in Dataset.filter( Dataset.external_filename.like( '%dataset_%.dat' ) ).all(): + if dataset.file_name in dataset_by_filename: + guessed_dataset = dataset_by_filename[ dataset.file_name ] + else: + guessed_dataset = __guess_dataset_by_filename( dataset.file_name ) + if guessed_dataset and dataset.file_name != guessed_dataset.file_name:#not os.path.samefile( dataset.file_name, guessed_dataset.file_name ): + guessed_dataset = None + dataset_by_filename[ dataset.file_name ] = guessed_dataset + + if guessed_dataset is not None and guessed_dataset.id != dataset.id: #could we have a self referential dataset? + for dataset_instance in dataset.history_associations + dataset.library_associations: + dataset_instance.dataset = guessed_dataset + changed_associations += 1 + #mark original Dataset as deleted and purged, it is no longer in use, but do not delete file_name contents + dataset.deleted = True + dataset.external_filename = "Dataset was result of share before HDA, and has been replaced: %s mapped to Dataset %s" % ( dataset.external_filename, guessed_dataset.id ) + dataset.purged = True #we don't really purge the file here, but we mark it as purged, since this dataset is now defunct + context.flush() + log.debug( "%i items affected, and restored." % ( changed_associations ) ) + log.debug( "Time elapsed: %s" % ( time.time() - start_time ) ) + +def downgrade(): + log.debug( "Downgrade is not possible." ) +