- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: jmchilton: Fix bug in tool watching.
by commits-noreply＠bitbucket.org 23 Oct '14

23 Oct '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0d9ac8314a78/ Changeset: 0d9ac8314a78 User: jmchilton Date: 2014-10-23 13:08:03+00:00 Summary: Fix bug in tool watching. Affected #: 1 file diff -r 53e734a1532733df32dcf45ace3dea6aa3f3604d -r 0d9ac8314a78087f68f84c66e861dffb9a1ea203 lib/galaxy/tools/watcher.py --- a/lib/galaxy/tools/watcher.py +++ b/lib/galaxy/tools/watcher.py @@ -51,7 +51,7 @@ self.tool_dir_callbacks[tool_dir] = callback if tool_dir not in self.monitored_dirs: self.monitored_dirs[ tool_dir ] = tool_dir - self.restart( tool_dir ) + self.monitor( tool_dir ) class ToolFileEventHandler(FileSystemEventHandler): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Implement the "memory_limit_reached" runner state for slurm and allow resubmission on that state.
by commits-noreply＠bitbucket.org 22 Oct '14

22 Oct '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/53e734a15327/ Changeset: 53e734a15327 User: natefoo Date: 2014-10-23 03:05:07+00:00 Summary: Implement the "memory_limit_reached" runner state for slurm and allow resubmission on that state. Affected #: 4 files diff -r 74499e753e0f28b4353a27e241c1fce0f0895964 -r 53e734a1532733df32dcf45ace3dea6aa3f3604d config/job_conf.xml.sample_advanced --- a/config/job_conf.xml.sample_advanced +++ b/config/job_conf.xml.sample_advanced @@ -399,8 +399,11 @@ the terminal condition of a job will be used. The 'condition' attribute is optional, if not present, the - resubmit destination will be used for all conditions. Currently, - only the "walltime_reached" condition is implemented. + resubmit destination will be used for all conditions. The + conditions currently implemented are: + + - "walltime_reached" + - "memory_limit_reached" The 'handler' tag is optional, if not present, the job's original handler will be reused for the resubmitted job. @@ -413,6 +416,10 @@ <param id="nativeSpecification">-l h_rt=96:00:00</param></destination> + <destination id="smallmem" runner="slurm"> + <param id="nativeSpecification">--mem-per-cpu=512</param> + <resubmit condition="memory_limit_reached" destination="bigmem" /> + </destination></destinations><resources default="default"> diff -r 74499e753e0f28b4353a27e241c1fce0f0895964 -r 53e734a1532733df32dcf45ace3dea6aa3f3604d lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -337,6 +337,7 @@ """ runner_states = Bunch( WALLTIME_REACHED = 'walltime_reached', + MEMORY_LIMIT_REACHED = 'memory_limit_reached', GLOBAL_WALLTIME_REACHED = 'global_walltime_reached', OUTPUT_SIZE_LIMIT = 'output_size_limit' ) diff -r 74499e753e0f28b4353a27e241c1fce0f0895964 -r 53e734a1532733df32dcf45ace3dea6aa3f3604d lib/galaxy/jobs/runners/slurm.py --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -13,6 +13,8 @@ __all__ = [ 'SlurmJobRunner' ] +SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit' + class SlurmJobRunner( DRMAAJobRunner ): runner_name = "SlurmRunner" @@ -62,8 +64,14 @@ except: ajs.fail_message = "This job failed due to a cluster node failure, and an attempt to resubmit the job failed." elif job_info['JobState'] == 'CANCELLED': - log.info( '(%s/%s) Job was cancelled via slurm (e.g. with scancel(1))', ajs.job_wrapper.get_id_tag(), ajs.job_id ) - ajs.fail_message = "This job failed because it was cancelled by an administrator." + # Check to see if the job was killed for exceeding memory consumption + if self.__check_memory_limit( ajs.error_file ): + log.info( '(%s/%s) Job hit memory limit', ajs.job_wrapper.get_id_tag(), ajs.job_id ) + ajs.fail_message = "This job was terminated because it used more memory than it was allocated." + ajs.runner_state = ajs.runner_states.MEMORY_LIMIT_REACHED + else: + log.info( '(%s/%s) Job was cancelled via slurm (e.g. with scancel(1))', ajs.job_wrapper.get_id_tag(), ajs.job_id ) + ajs.fail_message = "This job failed because it was cancelled by an administrator." else: log.warning( '(%s/%s) Job failed due to unknown reasons, JobState was: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, job_info['JobState'] ) ajs.fail_message = "This job failed for reasons that could not be determined." @@ -77,3 +85,31 @@ super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) # by default, finish as if the job was successful. super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) + + def __check_memory_limit( self, efile_path ): + """ + A very poor implementation of tail, but it doesn't need to be fancy + since we are only searching the last 2K + """ + try: + log.debug( 'Checking %s for exceeded memory message from slurm', efile_path ) + with open( efile_path ) as f: + pos = 2 + bof = False + while pos < 2048: + try: + f.seek(-pos, 2) + pos += 1 + except: + f.seek(-pos + 1, 2) + bof = True + + if (bof or f.read(1) == '\n') and f.readline().strip() == SLURM_MEMORY_LIMIT_EXCEEDED_MSG: + return True + + if bof: + break + except: + log.exception('Error reading end of %s:', path) + + return False diff -r 74499e753e0f28b4353a27e241c1fce0f0895964 -r 53e734a1532733df32dcf45ace3dea6aa3f3604d lib/galaxy/jobs/runners/state_handlers/resubmit.py --- a/lib/galaxy/jobs/runners/state_handlers/resubmit.py +++ b/lib/galaxy/jobs/runners/state_handlers/resubmit.py @@ -6,14 +6,24 @@ log = logging.getLogger(__name__) +MESSAGES = dict( + walltime_reached = 'it reached the walltime', + memory_limit_reached = 'it exceeded the amount of allocated memory' +) + def failure(app, job_runner, job_state): - if getattr( job_state, 'runner_state', None ) and job_state.runner_state == job_state.runner_states.WALLTIME_REACHED: + if getattr( job_state, 'runner_state', None ) and job_state.runner_state in ( job_state.runner_states.WALLTIME_REACHED, job_state.runner_states.MEMORY_LIMIT_REACHED ): # Intercept jobs that hit the walltime and have a walltime or nonspecific resubmit destination configured for resubmit in job_state.job_destination.get('resubmit'): - if resubmit.get('condition', None) and resubmit['condition'] != 'walltime_reached': + if resubmit.get('condition', None) and resubmit['condition'] != job_state.runner_state: continue # There is a resubmit defined for the destination but its condition is not for walltime_reached - log.info("(%s/%s) Job will be resubmitted to '%s' because it reached the walltime at the '%s' destination", job_state.job_wrapper.job_id, job_state.job_id, resubmit['destination'], job_state.job_wrapper.job_destination.id ) + log.info("(%s/%s) Job will be resubmitted to '%s' because %s at the '%s' destination", + job_state.job_wrapper.job_id, + job_state.job_id, + resubmit['destination'], + MESSAGES[job_state.runner_state], + job_state.job_wrapper.job_destination.id ) # fetch JobDestination for the id or tag new_destination = app.job_config.get_destination(resubmit['destination']) # Resolve dynamic if necessary Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Merge stable.
by commits-noreply＠bitbucket.org 22 Oct '14

22 Oct '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/74499e753e0f/ Changeset: 74499e753e0f User: dannon Date: 2014-10-23 02:52:11+00:00 Summary: Merge stable. Affected #: 1 file diff -r 7844d02f9c4731dcdbf78b99ba810805506a6c05 -r 74499e753e0f28b4353a27e241c1fce0f0895964 cron/updateucsc.sh.sample --- a/cron/updateucsc.sh.sample +++ b/cron/updateucsc.sh.sample @@ -4,8 +4,8 @@ # the update fails, not replace current data/tables with error # messages. -# Edit this line to refer to galaxy's path: -GALAXY=/galaxy/path +## Uncomment and edit this line to refer to galaxy's path: +#GALAXY=/galaxy/path PYTHONPATH=${GALAXY}/lib export PYTHONPATH Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Merged in BjoernGruening/galaxy-central/stable (pull request #537)
by commits-noreply＠bitbucket.org 22 Oct '14

22 Oct '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9bc08bc7f393/ Changeset: 9bc08bc7f393 Branch: stable User: dannon Date: 2014-10-23 02:50:21+00:00 Summary: Merged in BjoernGruening/galaxy-central/stable (pull request #537) [STABLE] Hide the GALAXY env variable. Affected #: 1 file diff -r ba1447685bc2470cb545612450b6364057b6d8c5 -r 9bc08bc7f393362ee7e6cf1d04bb9d8cee68f2d8 cron/updateucsc.sh.sample --- a/cron/updateucsc.sh.sample +++ b/cron/updateucsc.sh.sample @@ -4,8 +4,8 @@ # the update fails, not replace current data/tables with error # messages. -# Edit this line to refer to galaxy's path: -GALAXY=/galaxy/path +## Uncomment and edit this line to refer to galaxy's path: +#GALAXY=/galaxy/path PYTHONPATH=${GALAXY}/lib export PYTHONPATH Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 22 Oct '14

22 Oct '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/96eb54eec55b/ Changeset: 96eb54eec55b Branch: stable User: BjoernGruening Date: 2014-10-22 13:10:32+00:00 Summary: Hide the GALAXY env variable. This enables us to set it externally and just call the script. Very useful in docker containers. Affected #: 1 file diff -r 78c9a97be4b6bfe55136b055477fd1b9f22fd848 -r 96eb54eec55b362399fd9b6c856612218172983e cron/updateucsc.sh.sample --- a/cron/updateucsc.sh.sample +++ b/cron/updateucsc.sh.sample @@ -4,8 +4,8 @@ # the update fails, not replace current data/tables with error # messages. -# Edit this line to refer to galaxy's path: -GALAXY=/galaxy/path +## Uncomment and edit this line to refer to galaxy's path: +#GALAXY=/galaxy/path PYTHONPATH=${GALAXY}/lib export PYTHONPATH https://bitbucket.org/galaxy/galaxy-central/commits/9bc08bc7f393/ Changeset: 9bc08bc7f393 Branch: stable User: dannon Date: 2014-10-23 02:50:21+00:00 Summary: Merged in BjoernGruening/galaxy-central/stable (pull request #537) [STABLE] Hide the GALAXY env variable. Affected #: 1 file diff -r ba1447685bc2470cb545612450b6364057b6d8c5 -r 9bc08bc7f393362ee7e6cf1d04bb9d8cee68f2d8 cron/updateucsc.sh.sample --- a/cron/updateucsc.sh.sample +++ b/cron/updateucsc.sh.sample @@ -4,8 +4,8 @@ # the update fails, not replace current data/tables with error # messages. -# Edit this line to refer to galaxy's path: -GALAXY=/galaxy/path +## Uncomment and edit this line to refer to galaxy's path: +#GALAXY=/galaxy/path PYTHONPATH=${GALAXY}/lib export PYTHONPATH Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Merge stable.
by commits-noreply＠bitbucket.org 22 Oct '14

22 Oct '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7844d02f9c47/ Changeset: 7844d02f9c47 User: natefoo Date: 2014-10-23 02:47:52+00:00 Summary: Merge stable. Affected #: 2 files diff -r f25e90163bbd16626ce9adf7854ff493c79b469d -r 7844d02f9c4731dcdbf78b99ba810805506a6c05 .hgtags --- a/.hgtags +++ b/.hgtags @@ -20,4 +20,4 @@ ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 548ab24667d6206780237bd807f7d857a484c461 latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 -e1404e285ccb9e52bd09c5827934cd0da1974acc latest_2014.10.06 +a7b2aabb27ffe5cc3216774ab01b18f13b4fc941 latest_2014.10.06 diff -r f25e90163bbd16626ce9adf7854ff493c79b469d -r 7844d02f9c4731dcdbf78b99ba810805506a6c05 lib/galaxy/jobs/runners/slurm.py --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -19,13 +19,20 @@ def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): def __get_jobinfo(): - p = subprocess.Popen( ( 'scontrol', '-o', 'show', 'job', ajs.job_id ), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + job_id = ajs.job_id + cmd = [ 'scontrol', '-o' ] + if '.' in ajs.job_id: + # custom slurm-drmaa-with-cluster-support job id syntax + job_id, cluster = ajs.job_id.split('.', 1) + cmd.extend( [ '-M', cluster ] ) + cmd.extend( [ 'show', 'job', job_id ] ) + p = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = p.communicate() if p.returncode != 0: # Will need to be more clever here if this message is not consistent if stderr == 'slurm_load_jobs error: Invalid job id specified\n': return dict( JobState='NOT_FOUND' ) - raise Exception( '`scontrol -o show job %s` returned %s, stderr: %s' % ( ajs.job_id, p.returncode, stderr ) ) + raise Exception( '`%s` returned %s, stderr: %s' % ( ' '.join( cmd ), p.returncode, stderr ) ) return dict( [ out_param.split( '=', 1 ) for out_param in stdout.split() ] ) if drmaa_state == self.drmaa_job_states.FAILED: try: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Fix slurm job post-mortem for `clusters` functionality added to slurm-drmaa (and currently in use on usegalaxy.org).
by commits-noreply＠bitbucket.org 22 Oct '14

22 Oct '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ba1447685bc2/ Changeset: ba1447685bc2 Branch: stable User: natefoo Date: 2014-10-22 22:00:15+00:00 Summary: Fix slurm job post-mortem for `clusters` functionality added to slurm-drmaa (and currently in use on usegalaxy.org) Affected #: 1 file diff -r d1e1beeb532239250396dd8fbdc0156508c6744e -r ba1447685bc2470cb545612450b6364057b6d8c5 lib/galaxy/jobs/runners/slurm.py --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -19,13 +19,20 @@ def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): def __get_jobinfo(): - p = subprocess.Popen( ( 'scontrol', '-o', 'show', 'job', ajs.job_id ), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + job_id = ajs.job_id + cmd = [ 'scontrol', '-o' ] + if '.' in ajs.job_id: + # custom slurm-drmaa-with-cluster-support job id syntax + job_id, cluster = ajs.job_id.split('.', 1) + cmd.extend( [ '-M', cluster ] ) + cmd.extend( [ 'show', 'job', job_id ] ) + p = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = p.communicate() if p.returncode != 0: # Will need to be more clever here if this message is not consistent if stderr == 'slurm_load_jobs error: Invalid job id specified\n': return dict( JobState='NOT_FOUND' ) - raise Exception( '`scontrol -o show job %s` returned %s, stderr: %s' % ( ajs.job_id, p.returncode, stderr ) ) + raise Exception( '`%s` returned %s, stderr: %s' % ( ' '.join( cmd ), p.returncode, stderr ) ) return dict( [ out_param.split( '=', 1 ) for out_param in stdout.split() ] ) if drmaa_state == self.drmaa_job_states.FAILED: try: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] 3de753: Main: Schedule wig_to_bigWig and bed_to_bigBed mem...
by GitHub 22 Oct '14

22 Oct '14

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: 3de7534001170053915202c7e9ef50301a7b5eaf https://github.com/galaxyproject/usegalaxy-playbook/commit/3de7534001170053… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2014-10-22 (Wed, 22 Oct 2014) Changed paths: A files/galaxy/usegalaxy.org/dynamic_rules/single_dynamic_memory.py M production/group_vars/galaxyservers.yml M templates/galaxy/usegalaxy.org/config/job_conf.xml.j2 Log Message: ----------- Main: Schedule wig_to_bigWig and bed_to_bigBed memory dynamically.

1 0

[galaxyproject/usegalaxy-playbook] 007e0a: Test: Schedule wig_to_bigWig and bed_to_bigBed mem...
by GitHub 22 Oct '14

22 Oct '14

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: 007e0a78e74a558db22e1fc62fc1d8a81626fc33 https://github.com/galaxyproject/usegalaxy-playbook/commit/007e0a78e74a558d… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2014-10-22 (Wed, 22 Oct 2014) Changed paths: A files/galaxy/test.galaxyproject.org/dynamic_rules/single_dynamic_memory.py M stage/group_vars/galaxyservers.yml M templates/galaxy/test.galaxyproject.org/config/job_conf.xml.j2 Log Message: ----------- Test: Schedule wig_to_bigWig and bed_to_bigBed memory dynamically.

1 0

commit/galaxy-central: martenson: updated sphinx documentation: added managers; added new API controllers; cleanup
by commits-noreply＠bitbucket.org 22 Oct '14

22 Oct '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f25e90163bbd/ Changeset: f25e90163bbd User: martenson Date: 2014-10-22 21:55:45+00:00 Summary: updated sphinx documentation: added managers; added new API controllers; cleanup Affected #: 4 files diff -r 7c66e468c6024b2e17a70ffadae0ad080b174c68 -r f25e90163bbd16626ce9adf7854ff493c79b469d doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -11,7 +11,7 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys, os, datetime ####### REQUIRED GALAXY INCLUDES @@ -40,7 +40,7 @@ templates_path = ['_templates'] #configure default autodoc's action -autodoc_default_flags = ['members', 'undoc-members'] +autodoc_default_flags = [ 'members', 'undoc-members', 'show-inheritance' ] # The suffix of source filenames. source_suffix = '.rst' @@ -53,7 +53,7 @@ # General information about the project. project = u'Galaxy Code' -copyright = u'2012, Galaxy Team' +copyright = str( datetime.datetime.now().year ) + u', Galaxy Team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff -r 7c66e468c6024b2e17a70ffadae0ad080b174c68 -r f25e90163bbd16626ce9adf7854ff493c79b469d doc/source/lib/galaxy.managers.rst --- /dev/null +++ b/doc/source/lib/galaxy.managers.rst @@ -0,0 +1,92 @@ +managers Package +================== + +:mod:`managers` Package +------------------------- + +.. automodule:: galaxy.managers + + +:mod:`api_keys` Module +------------------------------ + +.. automodule:: galaxy.managers.api_keys + + +:mod:`base` Module +------------------------------ + +.. automodule:: galaxy.managers.base + + +:mod:`citations` Module +------------------------------ + +.. automodule:: galaxy.managers.citations + + +:mod:`collections` Module +------------------------------ + +.. automodule:: galaxy.managers.collections + + +:mod:`collections_util` Module +------------------------------ + +.. automodule:: galaxy.managers.collections_util + + +:mod:`context` Module +------------------------------ + +.. automodule:: galaxy.managers.context + + +:mod:`folders` Module +------------------------------ + +.. automodule:: galaxy.managers.folders + + +:mod:`hdas` Module +------------------------------ + +.. automodule:: galaxy.managers.hdas + + +:mod:`histories` Module +------------------------------ + +.. automodule:: galaxy.managers.histories + + +:mod:`lddas` Module +------------------------------ + +.. automodule:: galaxy.managers.lddas + + +:mod:`libraries` Module +------------------------------ + +.. automodule:: galaxy.managers.libraries + + +:mod:`roles` Module +------------------------------ + +.. automodule:: galaxy.managers.roles + + +:mod:`tags` Module +------------------------------ + +.. automodule:: galaxy.managers.tags + + +:mod:`workflows` Module +------------------------------ + +.. automodule:: galaxy.managers.workflows + :undoc-members: diff -r 7c66e468c6024b2e17a70ffadae0ad080b174c68 -r f25e90163bbd16626ce9adf7854ff493c79b469d doc/source/lib/galaxy.rst --- a/doc/source/lib/galaxy.rst +++ b/doc/source/lib/galaxy.rst @@ -38,6 +38,7 @@ galaxy.forms galaxy.jobs galaxy.model + galaxy.managers galaxy.objectstore galaxy.openid galaxy.quota @@ -51,4 +52,3 @@ galaxy.web galaxy.webapps galaxy.workflow - diff -r 7c66e468c6024b2e17a70ffadae0ad080b174c68 -r f25e90163bbd16626ce9adf7854ff493c79b469d doc/source/lib/galaxy.webapps.galaxy.api.rst --- a/doc/source/lib/galaxy.webapps.galaxy.api.rst +++ b/doc/source/lib/galaxy.webapps.galaxy.api.rst @@ -3,16 +3,16 @@ Background ========== -In addition to being accessible through a web interface, Galaxy can now also be +In addition to being accessible through a web interface, Galaxy can also be accessed programmatically, through shell scripts and other programs. The web interface is appropriate for things like exploratory analysis, visualization, construction of workflows, and rerunning workflows on new datasets. The web interface is less suitable for things like - Connecting a Galaxy instance directly to your sequencer and running - workflows whenever data is ready + workflows whenever data is ready. - Running a workflow against multiple datasets (which can be done with the - web interface, but is tedious) + web interface, but is tedious). - When the analysis involves complex control, such as looping and branching. @@ -132,7 +132,7 @@ description: API Test Library name: api_test - % ./display.py my_key http://localhost:4096/api/libraries/f3f73e481f432006/contents + % ./display.py my_key http://localhost:4096/api/libraries/f3f73e481f432006/contents Collection Members ------------------ /api/libraries/f3f73e481f432006/contents/28202595c0d2591f61ddda595d2c3670 @@ -205,208 +205,6 @@ The request and response format should be considered alpha and are subject to change. -API Return Codes and Formats -================== - -A set of error codes for API requests is being established and will be -documented here. This is a long-term project however so stayed tuned. - -API Controllers -=============== - -:mod:`datasets` Module ----------------------- - -.. automodule:: galaxy.webapps.galaxy.api.datasets - :members: - :undoc-members: - :show-inheritance: - -:mod:`folder_contents` Module ------------------------------ - -.. automodule:: galaxy.webapps.galaxy.api.folder_contents - :members: - :undoc-members: - :show-inheritance: - -:mod:`folders` Module ---------------------- - -.. automodule:: galaxy.webapps.galaxy.api.folders - :members: - :undoc-members: - :show-inheritance: - -:mod:`forms` Module -------------------- - -.. automodule:: galaxy.webapps.galaxy.api.forms - :members: - :undoc-members: - :show-inheritance: - -:mod:`genomes` Module ---------------------- - -.. automodule:: galaxy.webapps.galaxy.api.genomes - :members: - :undoc-members: - :show-inheritance: - -:mod:`group_roles` Module -------------------------- - -.. automodule:: galaxy.webapps.galaxy.api.group_roles - :members: - :undoc-members: - :show-inheritance: - -:mod:`group_users` Module -------------------------- - -.. automodule:: galaxy.webapps.galaxy.api.group_users - :members: - :undoc-members: - :show-inheritance: - -:mod:`groups` Module --------------------- - -.. automodule:: galaxy.webapps.galaxy.api.groups - :members: - :undoc-members: - :show-inheritance: - -:mod:`histories` Module ------------------------ - -.. automodule:: galaxy.webapps.galaxy.api.histories - :members: - :undoc-members: - :show-inheritance: - -:mod:`history_contents` Module ------------------------------- - -.. automodule:: galaxy.webapps.galaxy.api.history_contents - :members: - :undoc-members: - :show-inheritance: - -:mod:`item_tags` Module ------------------------ - -.. automodule:: galaxy.webapps.galaxy.api.item_tags - :members: - :undoc-members: - :show-inheritance: - -:mod:`lda_datasets` Module --------------------------- - -.. automodule:: galaxy.webapps.galaxy.api.lda_datasets - :members: - :undoc-members: - :show-inheritance: - -:mod:`libraries` Module ------------------------ - -.. automodule:: galaxy.webapps.galaxy.api.libraries - :members: - :undoc-members: - :show-inheritance: - -:mod:`library_contents` Module ------------------------------- - -.. automodule:: galaxy.webapps.galaxy.api.library_contents - :members: - :undoc-members: - :show-inheritance: - -:mod:`permissions` Module -------------------------- - -.. automodule:: galaxy.webapps.galaxy.api.permissions - :members: - :undoc-members: - :show-inheritance: - -:mod:`quotas` Module --------------------- - -.. automodule:: galaxy.webapps.galaxy.api.quotas - :members: - :undoc-members: - :show-inheritance: - -:mod:`request_types` Module ---------------------------- - -.. automodule:: galaxy.webapps.galaxy.api.request_types - :members: - :undoc-members: - :show-inheritance: - -:mod:`requests` Module ----------------------- - -.. automodule:: galaxy.webapps.galaxy.api.requests - :members: - :undoc-members: - :show-inheritance: - -:mod:`roles` Module -------------------- - -.. automodule:: galaxy.webapps.galaxy.api.roles - :members: - :undoc-members: - :show-inheritance: - -:mod:`samples` Module ---------------------- - -.. automodule:: galaxy.webapps.galaxy.api.samples - :members: - :undoc-members: - :show-inheritance: - -:mod:`tools` Module -------------------- - -.. automodule:: galaxy.webapps.galaxy.api.tools - :members: - :undoc-members: - :show-inheritance: - -:mod:`users` Module -------------------- - -.. automodule:: galaxy.webapps.galaxy.api.users - :members: - :undoc-members: - :show-inheritance: - -:mod:`visualizations` Module ----------------------------- - -.. automodule:: galaxy.webapps.galaxy.api.visualizations - :members: - :undoc-members: - :show-inheritance: - -:mod:`workflows` Module ------------------------ - -.. automodule:: galaxy.webapps.galaxy.api.workflows - :members: - :undoc-members: - :show-inheritance: - - API Design Guidelines ===================== @@ -418,26 +216,34 @@ - API functionality should include docstring documentation for consumption by readthedocs.org. - - Developers should familarize themselves with the HTTP status code definitions + - Developers should familiarize themselves with the HTTP status code definitions http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses should properly set the status code according to the result - in particular 2XX responses should be used for successful requests, 4XX for various - kinds of client errors, and 5XX for errors on the server side. + kinds of client errors, and 5XX for the errors on the server side. - If there is an error processing some part of request (one item in a list for instance), the status code should be set to reflect the error and the partial result may or may not be returned depending on the controller - this behavior should be documented. - - (TODO) API methods should throw a finite number of exceptions (defined in) - `galaxy.exceptions` and these should subclass `MessageException` and not - paste/wsgi HTTP exceptions. When possible, the framework itself should be - responsible catching these exceptions, setting the status code, and - building an error response. + - API methods should throw a finite number of exceptions + (defined in :doc:`galaxy.exceptions`) and these should subclass + `MessageException` and not paste/wsgi HTTP exceptions. When possible, + the framework itself should be responsible catching these exceptions, + setting the status code, and building an error response. - Error responses should not consist of plain text strings - they should be - dictionaries describing the error and containing the following keys (TODO: - spell out nature of this.) Various error conditions (once a format has - been chosen and framework to enforce it in place) should be spelled out - in this document. - - Backward compatibility is important and should maintained when possible. + dictionaries describing the error and containing the following:: + + { + "status_code": 400, + "err_code": 400007, + "err_msg": "Request contained invalid parameter, action could not be completed.", + "type": "error", + "extra_error_info": "Extra information." + } + + Various error conditions (once a format has been chosen and framework to + enforce it in place) should be spelled out in this document. + - Backward compatibility is important and should be maintained when possible. If changing behavior in a non-backward compatibile way please ensure one of the following holds - there is a strong reason to believe no consumers depend on a behavior, the behavior is effectively broken, or the API @@ -448,7 +254,8 @@ - Functionality should not be copied and pasted between controllers - consider refactoring functionality into associated classes or short of - that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance) + that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance) + or into Managers (:doc:`galaxy.managers`). - API additions are more permanent changes to Galaxy than many other potential changes and so a second opinion on API changes should be sought. (Consider a pull request!) @@ -457,16 +264,252 @@ `test/functional/api`. (Once such a framework is in place - it is not right now). - Changes to reflect modifications to the API should be pushed upstream to - the BioBlend project possible. + the BioBlend project if possible. Longer term goals/notes. - It would be advantageous to have a clearer separation of anonymous and admin handling functionality. - If at some point in the future, functionality needs to be added that - breaks backward compatibility in a significant way to a compontent used by - the community should be alerted - a "dev" variant of the API will be - established and the community should be alerted and given a timeframe - for when the old behavior will be replaced with the new behavior. + breaks backward compatibility in a significant way to a component used by + the community - a "dev" variant of the API will be established and + the community should be alerted and given a timeframe for when the old + behavior will be replaced with the new behavior. - Consistent standards for range-based requests, batch requests, filtered requests, etc... should be established and documented here. + +API Controllers +=============== + +Galaxy offers the following API controllers: + +:mod:`annotations` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.annotations + +:mod:`authenticate` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.authenticate + +:mod:`configuration` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.configuration + +:mod:`dataset_collections` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.dataset_collections + + +:mod:`datasets` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.datasets + + +:mod:`datatypes` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.datatypes + + +:mod:`extended_metadata` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.extended_metadata + + +:mod:`folder_contents` Module +----------------------------- + +.. automodule:: galaxy.webapps.galaxy.api.folder_contents + + +:mod:`folders` Module +--------------------- + +.. automodule:: galaxy.webapps.galaxy.api.folders + + +:mod:`forms` Module +------------------- + +.. automodule:: galaxy.webapps.galaxy.api.forms + + +:mod:`ftp_files` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.ftp_files + + +:mod:`genomes` Module +--------------------- + +.. automodule:: galaxy.webapps.galaxy.api.genomes + + +:mod:`group_roles` Module +------------------------- + +.. automodule:: galaxy.webapps.galaxy.api.group_roles + + +:mod:`group_users` Module +------------------------- + +.. automodule:: galaxy.webapps.galaxy.api.group_users + + +:mod:`groups` Module +-------------------- + +.. automodule:: galaxy.webapps.galaxy.api.groups + + +:mod:`histories` Module +----------------------- + +.. automodule:: galaxy.webapps.galaxy.api.histories + + +:mod:`history_contents` Module +------------------------------ + +.. automodule:: galaxy.webapps.galaxy.api.history_contents + + +:mod:`item_tags` Module +----------------------- + +.. automodule:: galaxy.webapps.galaxy.api.item_tags + + +:mod:`job_files` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.job_files + + +:mod:`jobs` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.jobs + + +:mod:`lda_datasets` Module +-------------------------- + +.. automodule:: galaxy.webapps.galaxy.api.lda_datasets + + +:mod:`libraries` Module +----------------------- + +.. automodule:: galaxy.webapps.galaxy.api.libraries + + +:mod:`library_contents` Module +------------------------------ + +.. automodule:: galaxy.webapps.galaxy.api.library_contents + + +:mod:`metrics` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.metrics + + +:mod:`page_revisions` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.page_revisions + + +:mod:`pages` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.pages + + +:mod:`provenance` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.provenance + + +:mod:`quotas` Module +-------------------- + +.. automodule:: galaxy.webapps.galaxy.api.quotas + + +:mod:`request_types` Module +--------------------------- + +.. automodule:: galaxy.webapps.galaxy.api.request_types + + +:mod:`requests` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.requests + + +:mod:`roles` Module +------------------- + +.. automodule:: galaxy.webapps.galaxy.api.roles + + +:mod:`samples` Module +--------------------- + +.. automodule:: galaxy.webapps.galaxy.api.samples + + +:mod:`search` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.search + + +:mod:`tool_data` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.tool_data + + +:mod:`tool_shed_repositories` Module +---------------------- + +.. automodule:: galaxy.webapps.galaxy.api.tool_shed_repositories + + +:mod:`tools` Module +------------------- + +.. automodule:: galaxy.webapps.galaxy.api.tools + + +:mod:`users` Module +------------------- + +.. automodule:: galaxy.webapps.galaxy.api.users + + +:mod:`visualizations` Module +---------------------------- + +.. automodule:: galaxy.webapps.galaxy.api.visualizations + + +:mod:`workflows` Module +----------------------- + +.. automodule:: galaxy.webapps.galaxy.api.workflows + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0