1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/c9cbd395ed49/
Changeset: c9cbd395ed49
User: dannon
Date: 2013-04-09 19:20:29
Summary: Add Parsley egg prior to merging search API.
Affected #: 1 file
diff -r af8a76870774ee664331dc018c4ad85cfe06b871 -r c9cbd395ed49dc8e4e211648c96ed1dcb5530aa4 eggs.ini
--- a/eggs.ini
+++ b/eggs.ini
@@ -45,6 +45,7 @@
nose = 0.11.1
NoseHTML = 0.4.1
NoseTestDiff = 0.1
+Parsley = 1.1
Paste = 1.7.5.1
PasteDeploy = 1.5.0
pexpect = 2.4
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/af8a76870774/
Changeset: af8a76870774
User: dan
Date: 2013-04-09 19:00:47
Summary: Change database logging of stderr and stdout to take text from start and end of the string (instead of just the start) when the size exceeds the set character limit (32k).
Affected #: 2 files
diff -r 2cfc5c8223ef102c320472cb84f197ca28a064d6 -r af8a76870774ee664331dc018c4ad85cfe06b871 lib/galaxy/jobs/__init__.py
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -34,6 +34,9 @@
# and should eventually become API'd
TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
+DATABASE_MAX_STRING_SIZE = 32768
+DATABASE_MAX_STRING_SIZE_PRETTY = '32K'
+
class Sleeper( object ):
"""
Provides a 'sleep' method that sleeps for a number of seconds *unless*
@@ -774,13 +777,13 @@
job.info = message
# TODO: Put setting the stdout, stderr, and exit code in one place
# (not duplicated with the finish method).
- if ( len( stdout ) > 32768 ):
- stdout = stdout[:32768]
- log.info( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id )
+ if ( len( stdout ) > DATABASE_MAX_STRING_SIZE ):
+ stdout = util.shrink_string_by_size( stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
+ log.info( "stdout for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
job.stdout = stdout
- if ( len( stderr ) > 32768 ):
- stderr = stderr[:32768]
- log.info( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id )
+ if ( len( stderr ) > DATABASE_MAX_STRING_SIZE ):
+ stderr = util.shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
+ log.info( "stderr for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
job.stderr = stderr
# Let the exit code be Null if one is not provided:
if ( exit_code != None ):
@@ -998,12 +1001,12 @@
# will now be seen by the user.
self.sa_session.flush()
# Save stdout and stderr
- if len( job.stdout ) > 32768:
- log.info( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id )
- job.stdout = job.stdout[:32768]
- if len( job.stderr ) > 32768:
- log.info( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id )
- job.stderr = job.stderr[:32768]
+ if len( job.stdout ) > DATABASE_MAX_STRING_SIZE:
+ log.info( "stdout for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
+ job.stdout = util.shrink_string_by_size( job.stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
+ if len( job.stderr ) > DATABASE_MAX_STRING_SIZE:
+ log.info( "stderr for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
+ job.stderr = util.shrink_string_by_size( job.stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
# The exit code will be null if there is no exit code to be set.
# This is so that we don't assign an exit code, such as 0, that
# is either incorrect or has the wrong semantics.
@@ -1652,12 +1655,12 @@
task.state = task.states.ERROR
# Save stdout and stderr
- if len( stdout ) > 32768:
- log.error( "stdout for task %d is greater than 32K, only first part will be logged to database" % task.id )
- task.stdout = stdout[:32768]
- if len( stderr ) > 32768:
- log.error( "stderr for job %d is greater than 32K, only first part will be logged to database" % task.id )
- task.stderr = stderr[:32768]
+ if len( stdout ) > DATABASE_MAX_STRING_SIZE:
+ log.error( "stdout for task %d is greater than %s, only a portion will be logged to database" % ( task.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
+ task.stdout = util.shrink_string_by_size( stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
+ if len( stderr ) > DATABASE_MAX_STRING_SIZE:
+ log.error( "stderr for task %d is greater than %s, only a portion will be logged to database" % ( task.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
+ task.stderr = util.shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
task.exit_code = tool_exit_code
task.command_line = self.command_line
self.sa_session.flush()
diff -r 2cfc5c8223ef102c320472cb84f197ca28a064d6 -r af8a76870774ee664331dc018c4ad85cfe06b871 lib/galaxy/util/__init__.py
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -152,6 +152,25 @@
elem.tail = i + pad
return elem
+def shrink_string_by_size( value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False ):
+ if len( value ) > size:
+ len_join_by = len( join_by )
+ min_size = len_join_by + 2
+ if size < min_size:
+ if beginning_on_size_error:
+ return value[:size]
+ elif end_on_size_error:
+ return value[-size:]
+ raise ValueError( 'With the provided join_by value (%s), the minimum size value is %i.' % ( join_by, min_size ) )
+ left_index = right_index = int( ( size - len_join_by ) / 2 )
+ if left_index + right_index + len_join_by < size:
+ if left_larger:
+ left_index += 1
+ else:
+ right_index += 1
+ value = "%s%s%s" % ( value[:left_index], join_by, value[-right_index:] )
+ return value
+
# characters that are valid
valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!")
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
2 new commits in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/d07c62f0067a/
Changeset: d07c62f0067a
User: dannon
Date: 2013-04-09 15:34:24
Summary: Fix bug from 8333 which prevented ToolDataTable loading .loc files from directories other than the default galaxy tool_data_path.
Affected #: 1 file
diff -r 8fc56b85e0a5353ffd6790685a4a7d8a84938409 -r d07c62f0067a4fc6cabab499d8c1191199c5fc8f lib/galaxy/tools/data/__init__.py
--- a/lib/galaxy/tools/data/__init__.py
+++ b/lib/galaxy/tools/data/__init__.py
@@ -55,7 +55,7 @@
self.data_table_elem_names.append( table_elem_name )
if from_shed_config:
self.shed_data_table_elems.append( table_elem )
- table = tool_data_table_types[ type ]( table_elem, tool_data_path )
+ table = tool_data_table_types[ type ]( table_elem, tool_data_path, from_shed_config)
if table.name not in self.data_tables:
self.data_tables[ table.name ] = table
log.debug( "Loaded tool data table '%s'", table.name )
@@ -132,7 +132,7 @@
os.chmod( full_path, 0644 )
class ToolDataTable( object ):
- def __init__( self, config_element, tool_data_path ):
+ def __init__( self, config_element, tool_data_path, from_shed_config = False):
self.name = config_element.get( 'name' )
self.comment_char = config_element.get( 'comment_char' )
self.empty_field_value = config_element.get( 'empty_field_value', '' )
@@ -164,11 +164,11 @@
type_key = 'tabular'
- def __init__( self, config_element, tool_data_path ):
- super( TabularToolDataTable, self ).__init__( config_element, tool_data_path )
- self.configure_and_load( config_element, tool_data_path )
+ def __init__( self, config_element, tool_data_path, from_shed_config = False):
+ super( TabularToolDataTable, self ).__init__( config_element, tool_data_path, from_shed_config)
+ self.configure_and_load( config_element, tool_data_path, from_shed_config)
- def configure_and_load( self, config_element, tool_data_path ):
+ def configure_and_load( self, config_element, tool_data_path, from_shed_config = False):
"""
Configure and load table from an XML element.
"""
@@ -180,7 +180,9 @@
all_rows = []
for file_element in config_element.findall( 'file' ):
found = False
- if tool_data_path:
+ if tool_data_path and from_shed_config:
+ # Must identify with from_shed_config as well, because the
+ # regular galaxy app has and uses tool_data_path.
# We're loading a tool in the tool shed, so we cannot use the Galaxy tool-data
# directory which is hard-coded into the tool_data_table_conf.xml entries.
filepath = file_element.get( 'path' )
https://bitbucket.org/galaxy/galaxy-central/commits/a395caa2f36f/
Changeset: a395caa2f36f
User: dannon
Date: 2013-04-09 15:36:23
Summary: Import cleanup, space methods, strip trailing whitespace.
Affected #: 1 file
diff -r d07c62f0067a4fc6cabab499d8c1191199c5fc8f -r a395caa2f36f2e4aa6c2daf24e4da3c9ede4b125 lib/galaxy/tools/data/__init__.py
--- a/lib/galaxy/tools/data/__init__.py
+++ b/lib/galaxy/tools/data/__init__.py
@@ -3,36 +3,47 @@
used by tools, for example in the generation of dynamic options. Tables are
loaded and stored by names which tools use to refer to them. This allows
users to configure data tables for a local Galaxy instance without needing
-to modify the tool configurations.
+to modify the tool configurations.
"""
-import logging, sys, os, os.path, tempfile, shutil
+import logging
+import os
+import os.path
+import shutil
+import tempfile
+
from galaxy import util
log = logging.getLogger( __name__ )
+
class ToolDataTableManager( object ):
"""Manages a collection of tool data tables"""
+
def __init__( self, tool_data_path, config_filename=None ):
self.tool_data_path = tool_data_path
# This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file
# at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries
# from that file are inserted into this dict at the time of installation.
- self.data_tables = {}
+ self.data_tables = {}
# Store config elements for on-the-fly persistence to the defined shed_tool_data_table_config file name.
self.shed_data_table_elems = []
self.data_table_elem_names = []
if config_filename:
self.load_from_config_file( config_filename, self.tool_data_path, from_shed_config=False )
+
def __getitem__( self, key ):
return self.data_tables.__getitem__( key )
+
def __contains__( self, key ):
return self.data_tables.__contains__( key )
+
def get( self, name, default=None ):
try:
return self[ name ]
except KeyError:
return default
+
def load_from_config_file( self, config_filename, tool_data_path, from_shed_config=False ):
"""
This method is called under 3 conditions:
@@ -65,6 +76,7 @@
if table_row not in self.data_tables[ table.name ].data:
self.data_tables[ table.name ].data.append( table_row )
return table_elems
+
def add_new_entries_from_config_file( self, config_filename, tool_data_path, shed_tool_data_table_config, persist=False ):
"""
This method is called when a tool shed repository that includes a tool_data_table_conf.xml.sample file is being
@@ -118,6 +130,7 @@
# Persist Galaxy's version of the changed tool_data_table_conf.xml file.
self.to_xml_file( shed_tool_data_table_config )
return table_elems, error_message
+
def to_xml_file( self, shed_tool_data_table_config ):
"""Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk."""
full_path = os.path.abspath( shed_tool_data_table_config )
@@ -130,8 +143,10 @@
os.close( fd )
shutil.move( filename, full_path )
os.chmod( full_path, 0644 )
-
+
+
class ToolDataTable( object ):
+
def __init__( self, config_element, tool_data_path, from_shed_config = False):
self.name = config_element.get( 'name' )
self.comment_char = config_element.get( 'comment_char' )
@@ -146,14 +161,16 @@
self.tool_data_file = None
self.tool_data_path = tool_data_path
self.missing_index_file = None
+
def get_empty_field_by_name( self, name ):
return self.empty_field_values.get( name, self.empty_field_value )
-
+
+
class TabularToolDataTable( ToolDataTable ):
"""
Data stored in a tabular / separated value format on disk, allows multiple
files to be merged but all must have the same column definitions::
-
+
<table type="tabular" name="test"><column name='...' index = '...' /><file path="..." />
@@ -161,9 +178,9 @@
</table>
"""
-
+
type_key = 'tabular'
-
+
def __init__( self, config_element, tool_data_path, from_shed_config = False):
super( TabularToolDataTable, self ).__init__( config_element, tool_data_path, from_shed_config)
self.configure_and_load( config_element, tool_data_path, from_shed_config)
@@ -224,8 +241,8 @@
with a name and index (as in dynamic options config), or a shorthand
comma separated list of names in order as the text of a 'column_names'
element.
-
- A column named 'value' is required.
+
+ A column named 'value' is required.
"""
self.columns = {}
if config_element.find( 'columns' ) is not None:
@@ -254,7 +271,7 @@
def parse_file_fields( self, reader ):
"""
Parse separated lines from file and return a list of tuples.
-
+
TODO: Allow named access to fields using the column names.
"""
separator_char = (lambda c: '<TAB>' if c == '\t' else c)(self.separator)
@@ -270,10 +287,10 @@
rval.append( fields )
else:
log.warn( "Line %i in tool data table '%s' is invalid (HINT: "
- "'%s' characters must be used to separate fields):\n%s"
+ "'%s' characters must be used to separate fields):\n%s"
% ( ( i + 1 ), self.name, separator_char, line ) )
return rval
-
+
def get_column_name_list( self ):
rval = []
for i in range( self.largest_index + 1 ):
@@ -286,7 +303,7 @@
if not found_column:
rval.append( None )
return rval
-
+
def get_entry( self, query_attr, query_val, return_attr, default=None ):
"""
Returns table entry associated with a col/val pair.
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.