3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c8e18e05f0ec/ Changeset: c8e18e05f0ec User: jmchilton Date: 2014-05-05 16:11:05 Summary: Truncate big job metric values. Wasn't needed for sqlite based testing - but based on Nicola's experience must be for some other database backend. https://bitbucket.org/galaxy/galaxy-central/commits/a45f6e57f3d5a4075430a6d4... Affected #: 2 files diff -r 92519a9bfa32a42ce47a63fdb72c8e2d717ec52e -r c8e18e05f0ecee85cc6d3d7122b457fbc85915a4 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -107,6 +107,10 @@ else: if isinstance( metric_value, str ): metric_value = unicode( metric_value, 'utf-8' ) + if len( metric_value ) > 1022: + # Truncate these values - not needed with sqlite + # but other backends must need it. + metric_value = metric_value[ :1022 ] metric = self._text_metric( plugin, metric_name, metric_value ) self.text_metrics.append( metric ) diff -r 92519a9bfa32a42ce47a63fdb72c8e2d717ec52e -r c8e18e05f0ecee85cc6d3d7122b457fbc85915a4 test/unit/test_galaxy_mapping.py --- a/test/unit/test_galaxy_mapping.py +++ b/test/unit/test_galaxy_mapping.py @@ -237,7 +237,12 @@ task = model.Task( job=job, working_directory="/tmp", prepare_files_cmd="split.sh" ) task.add_metric( "gx", "galaxy_slots", 5 ) task.add_metric( "system", "system_name", "localhost" ) + + big_value = ":".join( [ "%d" % i for i in range( 2000 ) ] ) + task.add_metric( "env", "BIG_PATH", big_value ) self.persist( task ) + # Ensure big values truncated + assert len( task.text_metrics[ 1 ].metric_value ) <= 1023 def test_tasks( self ): model = self.model https://bitbucket.org/galaxy/galaxy-central/commits/d9c6648ef265/ Changeset: d9c6648ef265 User: jmchilton Date: 2014-05-05 16:11:05 Summary: Use Nicola's improved logic for env output parsing for env metrics plugin. Very well laid out here https://bitbucket.org/galaxy/galaxy-central/commits/a45f6e57f3d5a4075430a6d4... with examples. The one modification I made was to move the logic for truncated large property values into the model layer so it applies for all plugins. Affected #: 1 file diff -r c8e18e05f0ecee85cc6d3d7122b457fbc85915a4 -r d9c6648ef2658cc06708ecf908f8b5ce88a7996f lib/galaxy/jobs/metrics/instrumenters/env.py --- a/lib/galaxy/jobs/metrics/instrumenters/env.py +++ b/lib/galaxy/jobs/metrics/instrumenters/env.py @@ -1,3 +1,5 @@ +import re + from ..instrumenters import InstrumentPlugin from ...metrics import formatting @@ -41,17 +43,19 @@ variables = self.variables properties = {} - for line in open( self.__env_file( job_directory ) ).readlines(): - if "=" not in line: - # Previous line may have had a multiline property value, just - # keep it simple here and only record the first part of - # property. A more robust solution might be to record env -O - # so properties are terminated by null characters instead of - # newlines. - continue - var, value = line.split( "=", 1 ) + env_string = ''.join( open( self.__env_file( job_directory ) ).readlines() ) + while env_string: + # Check if the next lines contain a shell function. + # We use '\n\}\n' as regex termination because shell + # functions can be nested. + # We use the non-greedy '.+?' because of re.DOTALL . + m = re.match( '([^=]+)=(\(\) \{.+?\n\})\n', env_string, re.DOTALL ) + if m is None: + m = re.match( '([^=]+)=(.*)\n', env_string ) + (var, value) = m.groups() if not variables or var in variables: properties[ var ] = value + env_string = env_string[m.end():] return properties https://bitbucket.org/galaxy/galaxy-central/commits/848d2c4e41c2/ Changeset: 848d2c4e41c2 User: jmchilton Date: 2014-05-05 16:11:05 Summary: Slightly more defensive job metrics env parsing. The env string potentially may not match exactly if there are problems reading or recording env output. Log the problem and leave the parsing loop. Affected #: 1 file diff -r d9c6648ef2658cc06708ecf908f8b5ce88a7996f -r 848d2c4e41c27e67e07431b152b0b329676ab8c5 lib/galaxy/jobs/metrics/instrumenters/env.py --- a/lib/galaxy/jobs/metrics/instrumenters/env.py +++ b/lib/galaxy/jobs/metrics/instrumenters/env.py @@ -52,6 +52,12 @@ m = re.match( '([^=]+)=(\(\) \{.+?\n\})\n', env_string, re.DOTALL ) if m is None: m = re.match( '([^=]+)=(.*)\n', env_string ) + if m is None: + # Some problem recording or reading back env output. + message_template = "Problem parsing env metric output for job %s - properties will be incomplete" + message = message_template % job_id + log.debug( message ) + break (var, value) = m.groups() if not variables or var in variables: properties[ var ] = value Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.