1 new changeset in galaxy-central:
http://bitbucket.org/galaxy/galaxy-central/changeset/21cdfab7b813/
changeset: r5035:21cdfab7b813
user: natefoo
date: 2011-02-09 22:46:30
summary: A fix for DRMs which can apparently cause a drmaa.InternalException in cases where the job is finished normally. If there are other conditions which raise this exception, failures may go undetected. The drmaa runner is not in active use by the development team, so please report problems with this change.
affected #: 1 file (278 bytes)
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new changeset in galaxy-central:
http://bitbucket.org/galaxy/galaxy-central/changeset/b69b84f67337/
changeset: r5034:b69b84f67337
user: natefoo
date: 2011-02-09 22:31:43
summary: More comprehensive conflict resolution for eggs that are dependencies of other eggs, and eggs which conflict with system versions.
affected #: 1 file (1.8 KB)
--- a/lib/galaxy/eggs/__init__.py Wed Feb 09 16:29:50 2011 -0500
+++ b/lib/galaxy/eggs/__init__.py Wed Feb 09 16:31:43 2011 -0500
@@ -36,12 +36,13 @@
"""
Contains information about locating and downloading eggs.
"""
- def __init__( self, name=None, version=None, tag=None, url=None, platform=None ):
+ def __init__( self, name=None, version=None, tag=None, url=None, platform=None, crate=None ):
self.name = name
self.version = version
self.tag = tag
self.url = url
self.platform = platform
+ self.crate = crate
self.distribution = None
self.dir = None
if self.name is not None and self.version is not None:
@@ -145,37 +146,68 @@
log.debug( "Removed conflicting egg: %s" % doppelganger )
def resolve( self ):
try:
- return pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch )
+ rval = []
+ # resolve this egg and its dependencies
+ dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch )
+ for dist in dists:
+ # if any of the resolved dependencies should be managed eggs but are being pulled from the wrong path, fix them
+ if dist.project_name in self.crate.all_names and not os.path.realpath( dist.location ).startswith( os.path.realpath( self.dir ) ):
+ # TODO: like eggs.require(), this breaks requirement versioning
+ subdists = self.version_conflict( dist, dist.as_requirement() )
+ if type( subdists ) == list:
+ rval.extend( subdists )
+ else:
+ rval.append( subdists )
+ else:
+ rval.append( dist )
+ return rval
except pkg_resources.DistributionNotFound, e:
# If this statement is true, it means we do have the requested egg,
# just not one (or more) of its deps.
if e.args[0].project_name != self.distribution.project_name:
- log.warning( "Warning: %s (a dependant egg of %s) cannot be fetched" % ( e.args[0].project_name, self.distribution.project_name ) )
+ log.warning( "Warning: %s (a dependent egg of %s) cannot be fetched" % ( e.args[0].project_name, self.distribution.project_name ) )
return ( self.distribution, )
else:
raise EggNotFetchable( self )
except pkg_resources.VersionConflict, e:
# there's a conflicting egg on the path, remove it
- dist = e.args[0]
- # use the canonical path for comparisons
- location = os.path.realpath( dist.location )
- for entry in pkg_resources.working_set.entries:
- if os.path.realpath( entry ) == location:
- pkg_resources.working_set.entries.remove( entry )
- break
- else:
- location = entry = None
- del pkg_resources.working_set.by_key[dist.key]
- if entry is not None:
- pkg_resources.working_set.entry_keys[entry] = []
- if entry in sys.path:
- sys.path.remove(entry)
- r = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch )
- if location is not None and not location.endswith( '.egg' ):
- # re-add the path if it's a non-egg dir, in case more deps live there
- pkg_resources.working_set.entries.append( location )
- sys.path.append( location )
- return r
+ return self.version_conflict( e.args[0], e.args[1] )
+ def version_conflict( self, conflict_dist, conflict_req ):
+ # since this conflict may be for a dependent egg, find the correct egg from the crate
+ if conflict_dist.project_name == self.distribution.project_name:
+ egg = self
+ dist = egg.distribution
+ elif conflict_dist.project_name in self.crate.all_names:
+ egg = self.crate[conflict_dist.project_name]
+ dist = egg.distribution
+ else:
+ # should not happen, but just in case
+ egg = None
+ dist = conflict_dist
+ # use the canonical path to locate and remove the conflict from the working set
+ location = os.path.realpath( conflict_dist.location )
+ for entry in pkg_resources.working_set.entries:
+ if os.path.realpath( entry ) == location:
+ pkg_resources.working_set.entries.remove( entry )
+ break
+ else:
+ location = entry = None
+ del pkg_resources.working_set.by_key[conflict_dist.key]
+ # remove the conflict from sys.path
+ if entry is not None:
+ pkg_resources.working_set.entry_keys[entry] = []
+ if entry in sys.path:
+ sys.path.remove(entry)
+ # if the conflict is a dpeendent egg, fetch that specific egg
+ if egg:
+ r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch )
+ else:
+ r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env )
+ # re-add the path if it's a non-egg dir, in case more deps live there
+ if location is not None and not location.endswith( '.egg' ):
+ pkg_resources.working_set.entries.append( location )
+ sys.path.append( location )
+ return r
def require( self ):
try:
dists = self.resolve()
@@ -215,7 +247,7 @@
platform = self.platform or '-'.join( ( py, pkg_resources.get_platform() ) )
else:
platform = self.py_platform or py
- egg = egg_class( name, version, tag, url, platform )
+ egg = egg_class( name=name, version=version, tag=tag, url=url, platform=platform, crate=self )
self.eggs[name] = egg
@property
def config_missing( self ):
@@ -331,7 +363,7 @@
return False
def get_env():
- env = pkg_resources.Environment( platform=pkg_resources.get_platform() )
+ env = pkg_resources.Environment( search_path='', platform=pkg_resources.get_platform() )
for dist in pkg_resources.find_distributions( os.path.join( galaxy_dir, 'eggs' ), False ):
env.add( dist )
return env
@@ -340,6 +372,8 @@
def require( req_str ):
c = Crate()
req = pkg_resources.Requirement.parse( req_str )
+ # TODO: This breaks egg version requirements. Not currently a problem, but
+ # it could become one.
try:
return c[req.project_name].require()
except KeyError:
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new changeset in galaxy-central:
http://bitbucket.org/galaxy/galaxy-central/changeset/c366f4f232ea/
changeset: r5032:c366f4f232ea
user: greg
date: 2011-02-09 22:18:56
summary: Optimize the printed output of the 0074... migration script.
affected #: 1 file (29 bytes)
--- a/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py Wed Feb 09 15:42:42 2011 -0500
+++ b/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py Wed Feb 09 16:18:56 2011 -0500
@@ -48,10 +48,10 @@
cmd = "SELECT * FROM library_dataset WHERE deleted = %s;" % boolean_true()
deleted_lds = db_session.execute( cmd ).fetchall()
for row in deleted_lds:
- print "Processing LibraryDataset id: ", int( row.id )
cmd = "SELECT * FROM library_dataset_dataset_association WHERE library_dataset_id = %d AND library_dataset_dataset_association.deleted = %s;" % ( int( row.id ), boolean_false() )
active_lddas = db_session.execute( cmd ).fetchall()
if not active_lddas:
+ print "Updating purged column to True for LibraryDataset id : ", int( row.id )
cmd = "UPDATE library_dataset SET purged = %s WHERE id = %d;" % ( boolean_true(), int( row.id ) )
db_session.execute( cmd )
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new changeset in galaxy-central:
http://bitbucket.org/galaxy/galaxy-central/changeset/909e158e2e3a/
changeset: r5031:909e158e2e3a
user: dan
date: 2011-02-09 21:42:42
summary: Add metabolicMine datasource tool.
affected #: 2 files (50 bytes)
--- a/tool_conf.xml.sample Wed Feb 09 15:42:09 2011 -0500
+++ b/tool_conf.xml.sample Wed Feb 09 15:42:42 2011 -0500
@@ -17,6 +17,7 @@
<tool file="data_source/modmine.xml" /><tool file="data_source/ratmine.xml" /><tool file="data_source/yeastmine.xml" />
+ <tool file="data_source/metabolicmine.xml" /><tool file="data_source/worm_modencode.xml" /><tool file="data_source/wormbase.xml" /><tool file="data_source/wormbase_test.xml" />
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new changeset in galaxy-central:
http://bitbucket.org/galaxy/galaxy-central/changeset/2626e0f79dc2/
changeset: r5030:2626e0f79dc2
user: greg
date: 2011-02-09 21:42:09
summary: Add a purged column to the LibraryDataset table - a LibraryDataset is marked purged when all associated LibraryDatasetDatasetAssociations are marked deleted. Fix the cleanup_datasets.py script to more correclty handle the lifecycle of LibraryDatasets.
affected #: 4 files (4.3 KB)
--- a/lib/galaxy/model/__init__.py Wed Feb 09 13:24:35 2011 -0500
+++ b/lib/galaxy/model/__init__.py Wed Feb 09 15:42:09 2011 -0500
@@ -1106,14 +1106,6 @@
name = property( get_name, set_name )
def display_name( self ):
self.library_dataset_dataset_association.display_name()
- def get_purged( self ):
- return self.library_dataset_dataset_association.dataset.purged
- def set_purged( self, purged ):
- if purged:
- raise Exception( "Not implemented" )
- if not purged and self.purged:
- raise Exception( "Cannot unpurge once purged" )
- purged = property( get_purged, set_purged )
def get_api_value( self, view='collection' ):
# Since this class is a proxy to rather complex attributes we want to
# display in other objects, we can't use the simpler method used by
--- a/lib/galaxy/model/mapping.py Wed Feb 09 13:24:35 2011 -0500
+++ b/lib/galaxy/model/mapping.py Wed Feb 09 15:42:09 2011 -0500
@@ -255,7 +255,8 @@
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), key="_name", index=True ), #when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "info", TrimmedString( 255 ), key="_info" ), #when not None/null this will supercede display in library (but not when imported into user's history?)
- Column( "deleted", Boolean, index=True, default=False ) )
+ Column( "deleted", Boolean, index=True, default=False ),
+ Column( "purged", Boolean, index=True, default=False ) )
LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
--- a/scripts/cleanup_datasets/cleanup_datasets.py Wed Feb 09 13:24:35 2011 -0500
+++ b/scripts/cleanup_datasets/cleanup_datasets.py Wed Feb 09 15:42:09 2011 -0500
@@ -22,6 +22,43 @@
assert sys.version_info[:2] >= ( 2, 4 )
def main():
+ """
+ Managing library datasets is a bit complex, so here is a scenario that hopefully provides clarification. The complexities
+ of handling library datasets is mostly contained in the delete_datasets() method in this script.
+
+ Assume we have 1 library dataset with: LibraryDatasetDatasetAssociation -> LibraryDataset and Dataset
+ At this point, we have the following database column values:
+
+ LibraryDatasetDatasetAssociation deleted: False
+ LibraryDataset deleted: False, purged: False
+ Dataset deleted: False purged: False
+
+ 1. A user deletes the assumed dataset above from a data library via a UI menu option.
+ This action results in the following database column values (changes from previous step marked with *):
+
+ LibraryDatasetDatasetAssociation deleted: False
+ LibraryDataset deleted: True*, purged: False
+ Dataset deleted: False, purged: False
+
+ 2. After the number of days configured for the delete_datasets() method (option -6 below) have passed, execution
+ of the delete_datasets() method results in the following database column values (changes from previous step marked with *):
+
+ LibraryDatasetDatasetAssociation deleted: True*
+ LibraryDataset deleted: True, purged: True*
+ Dataset deleted: True*, purged: False
+
+ 3. After the number of days configured for the purge_datasets() method (option -3 below) have passed, execution
+ of the purge_datasets() method results in the following database column values (changes from previous step marked with *):
+
+ LibraryDatasetDatasetAssociation deleted: True
+ LibraryDataset deleted: True, purged: True
+ Dataset deleted: True, purged: True* (dataset file removed from disk if -r flag is used)
+
+ This scenario is about as simple as it gets. Keep in mind that a Dataset object can have many HistoryDatasetAssociations
+ and many LibraryDatasetDatasetAssociations, and a LibraryDataset can have many LibraryDatasetDatasetAssociations.
+ Another way of stating it is: LibraryDatasetDatasetAssociation objects map LibraryDataset objects to Dataset objects,
+ and Dataset objects may be mapped to History objects via HistoryDatasetAssociation objects.
+ """
parser = OptionParser()
parser.add_option( "-d", "--days", dest="days", action="store", type="int", help="number of days (60)", default=60 )
parser.add_option( "-r", "--remove_from_disk", action="store_true", dest="remove_from_disk", help="remove datasets from disk when purged", default=False )
@@ -215,40 +252,70 @@
whereclause = app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
from_obj = [ sa.outerjoin( app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table ) ] )
- library_dataset_ids_query = sa.select( ( app.model.Dataset.table.c.id,
- app.model.Dataset.table.c.state ),
- whereclause = app.model.LibraryDatasetDatasetAssociation.table.c.update_time < cutoff_time,
- from_obj = [ sa.outerjoin( app.model.Dataset.table,
- app.model.LibraryDatasetDatasetAssociation.table ) ] )
+ library_dataset_ids_query = sa.select( ( app.model.LibraryDataset.table.c.id,
+ app.model.LibraryDataset.table.c.deleted ),
+ whereclause = app.model.LibraryDataset.table.c.update_time < cutoff_time,
+ from_obj = [ app.model.LibraryDataset.table ] )
else:
# We really only need the id column here, but sqlalchemy barfs when trying to select only 1 column
history_dataset_ids_query = sa.select( ( app.model.Dataset.table.c.id,
app.model.Dataset.table.c.state ),
- whereclause = sa.and_( app.model.Dataset.table.c.deleted == False,
- app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
- app.model.HistoryDatasetAssociation.table.c.deleted == True ),
+ whereclause = and_( app.model.Dataset.table.c.deleted == False,
+ app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
+ app.model.HistoryDatasetAssociation.table.c.deleted == True ),
from_obj = [ sa.outerjoin( app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table ) ] )
- library_dataset_ids_query = sa.select( ( app.model.Dataset.table.c.id,
- app.model.Dataset.table.c.state ),
- whereclause = sa.and_( app.model.Dataset.table.c.deleted == False,
- app.model.LibraryDatasetDatasetAssociation.table.c.update_time < cutoff_time,
- app.model.LibraryDatasetDatasetAssociation.table.c.deleted == True ),
- from_obj = [ sa.outerjoin( app.model.Dataset.table,
- app.model.LibraryDatasetDatasetAssociation.table ) ] )
- history_dataset_ids = [ row.id for row in history_dataset_ids_query.execute() ]
- library_dataset_ids = [ row.id for row in library_dataset_ids_query.execute() ]
- dataset_ids = history_dataset_ids + library_dataset_ids
- skip = []
+ library_dataset_ids_query = sa.select( ( app.model.LibraryDataset.table.c.id,
+ app.model.LibraryDataset.table.c.deleted ),
+ whereclause = and_( app.model.LibraryDataset.table.c.deleted == True,
+ app.model.LibraryDataset.table.c.purged == False,
+ app.model.LibraryDataset.table.c.update_time < cutoff_time ),
+ from_obj = [ app.model.LibraryDataset.table ] )
deleted_dataset_count = 0
deleted_instance_count = 0
+ skip = []
+ # Handle library datasets. This is a bit tricky, so here's some clarification. We have a list of all
+ # LibraryDatasets that were marked deleted before our cutoff_time, but have not yet been marked purged.
+ # A LibraryDataset object is marked purged when all of it's LibraryDatasetDatasetAssociations have been
+ # marked deleted. When a LibraryDataset has been marked purged, it can never be undeleted in the data
+ # library. We have several steps to complete here. For each LibraryDataset, get it's associated Dataset
+ # and add it to our accrued list of Datasets for later processing. We mark as deleted all of it's
+ # LibraryDatasetDatasetAssociations. Then we mark the LibraryDataset as purged. We then process our
+ # list of Datasets.
+ library_dataset_ids = [ row.id for row in library_dataset_ids_query.execute() ]
+ dataset_ids = []
+ for library_dataset_id in library_dataset_ids:
+ print "######### Processing LibraryDataset id:", library_dataset_id
+ # Get the LibraryDataset and the current LibraryDatasetDatasetAssociation objects
+ ld = app.sa_session.query( app.model.LibraryDataset ).get( library_dataset_id )
+ ldda = ld.library_dataset_dataset_association
+ # Append the associated Dataset object's id to our list of dataset_ids
+ dataset_ids.append( ldda.dataset_id )
+ # Mark all of the LibraryDataset's associated LibraryDatasetDatasetAssociation objects' as deleted
+ if not ldda.deleted:
+ ldda.deleted = True
+ app.sa_session.add( ldda )
+ print "Marked associated LibraryDatasetDatasetAssociation id %d as deleted" % ldda.id
+ for expired_ldda in ld.expired_datasets:
+ if not expired_ldda.deleted:
+ expired_ldda.deleted = True
+ app.sa_session.add( expired_ldda )
+ print "Marked associated expired LibraryDatasetDatasetAssociation id %d as deleted" % ldda.id
+ # Mark the LibraryDataset as purged
+ ld.purged = True
+ app.sa_session.add( ld )
+ print "Marked LibraryDataset id %d as purged" % ld.id
+ app.sa_session.flush()
+ # Add all datasets associated with Histories to our list
+ dataset_ids.extend( [ row.id for row in history_dataset_ids_query.execute() ] )
+ # Process each of the Dataset objects
for dataset_id in dataset_ids:
print "######### Processing dataset id:", dataset_id
dataset = app.sa_session.query( app.model.Dataset ).get( dataset_id )
if dataset.id not in skip and _dataset_is_deletable( dataset ):
deleted_dataset_count += 1
for dataset_instance in dataset.history_associations + dataset.library_associations:
- print "Associated Dataset instance: ", dataset_instance.__class__.__name__, dataset_instance.id
+ # Mark each associated HDA as deleted
_purge_dataset_instance( dataset_instance, app, remove_from_disk, include_children=True, info_only=info_only, is_deletable=True )
deleted_instance_count += 1
skip.append( dataset.id )
@@ -293,7 +360,7 @@
# A dataset_instance is either a HDA or an LDDA. Purging a dataset instance marks the instance as deleted,
# and marks the associated dataset as deleted if it is not associated with another active DatsetInstance.
if not info_only:
- print "Deleting dataset_instance ", str( dataset_instance ), " id ", dataset_instance.id
+ print "Marking as deleted: ", dataset_instance.__class__.__name__, " id ", dataset_instance.id
dataset_instance.mark_deleted( include_children = include_children )
dataset_instance.clear_associated_files()
app.sa_session.add( dataset_instance )
@@ -312,8 +379,8 @@
return not bool( dataset.active_history_associations or dataset.active_library_associations )
def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletable=False ):
- #marks a base dataset as deleted, hdas/ldas associated with dataset can no longer be undeleted
- #metadata files attached to associated dataset Instances is removed now
+ # Marks a base dataset as deleted, hdas/lddas associated with dataset can no longer be undeleted.
+ # Metadata files attached to associated dataset Instances is removed now.
if not is_deletable and not _dataset_is_deletable( dataset ):
print "This Dataset (%i) is not deletable, associated Metadata Files will not be removed.\n" % ( dataset.id )
else:
@@ -324,9 +391,9 @@
for metadata_file in app.sa_session.query( app.model.MetadataFile ) \
.filter( app.model.MetadataFile.table.c.hda_id==hda.id ):
metadata_files.append( metadata_file )
- for lda in dataset.library_associations:
+ for ldda in dataset.library_associations:
for metadata_file in app.sa_session.query( app.model.MetadataFile ) \
- .filter( app.model.MetadataFile.table.c.lda_id==lda.id ):
+ .filter( app.model.MetadataFile.table.c.lda_id==ldda.id ):
metadata_files.append( metadata_file )
for metadata_file in metadata_files:
print "The following metadata files attached to associations of Dataset '%s' have been purged:" % dataset.id
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new changeset in galaxy-central:
http://bitbucket.org/galaxy/galaxy-central/changeset/397c31931ce5/
changeset: r5029:397c31931ce5
user: kanwei
date: 2011-02-09 19:24:35
summary: trackster: Recalc slots no matter what level it's on since it's fast enough
affected #: 1 file (85 bytes)
--- a/static/scripts/trackster.js Wed Feb 09 13:11:28 2011 -0500
+++ b/static/scripts/trackster.js Wed Feb 09 13:24:35 2011 -0500
@@ -2321,9 +2321,8 @@
}
// Calculate new slots incrementally for this new chunk of data and update height if necessary.
- var inc_scale = (w_scale < 0.0001 ? 1/this.view.zoom_res : w_scale);
- required_height = this.incremental_slots(inc_scale, result.data, no_label, mode) * y_scale + min_height;
- slots = this.inc_slots[inc_scale];
+ required_height = this.incremental_slots(w_scale, result.data, no_label, mode) * y_scale + min_height;
+ slots = this.inc_slots[w_scale];
}
canvas.get(0).width = width + left_offset;
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new changeset in galaxy-central:
http://bitbucket.org/galaxy/galaxy-central/changeset/249962c12b39/
changeset: r5028:249962c12b39
user: kellyv
date: 2011-02-09 19:11:28
summary: Added bwa_index_color.loc to buildbot setup config
affected #: 1 file (42 bytes)
--- a/buildbot_setup.sh Wed Feb 09 12:36:40 2011 -0500
+++ b/buildbot_setup.sh Wed Feb 09 13:11:28 2011 -0500
@@ -39,6 +39,7 @@
/galaxy/data/location/bowtie_indices.loc
/galaxy/data/location/bowtie_indices_color.loc
/galaxy/data/location/bwa_index.loc
+/galaxy/data/location/bwa_index_color.loc
/galaxy/data/location/codingSnps.loc
/galaxy/data/location/encode_datasets.loc
/galaxy/home/universe/encode_feature_partitions
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.