- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 28 Apr '14

28 Apr '14

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9dd03dd46d04/ Changeset: 9dd03dd46d04 User: jmchilton Date: 2014-04-28 18:48:51 Summary: Create run_tests.sh to replace run_functional_tests.sh Should be backward compatible but features better argument handling - things can appear in other orders (-id x -data_managers is the same as -data_managers -id x for instance), better error messages if arguments are missing, and more standard short argument with single dash or long argument with double dash variants of each argument are available. Affected #: 1 file diff -r 4a6cd1da30d0c4e3f8e0ed7b1c1e6342bed20e44 -r 9dd03dd46d04b83aeb270ebe538d6075facb5379 run_tests.sh --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,149 @@ +#!/bin/sh + +# A good place to look for nose info: http://somethingaboutorange.com/mrl/projects/nose/ +rm -f run_functional_tests.log + +show_help() { +cat <<EOF +'${0##*/}' for testing all the tools in functional directory +'${0##*/} aaa' for testing one test case of 'aaa' ('aaa' is the file name with path) +'${0##*/} -id bbb' for testing one tool with id 'bbb' ('bbb' is the tool id) +'${0##*/} -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::') +'${0##*/} -list' for listing all the tool ids +'${0##*/} -toolshed' for running all the test scripts in the ./test/tool_shed/functional directory +'${0##*/} -toolshed testscriptname' for running one test script named testscriptname in the .test/tool_shed/functional directory +'${0##*/} -workflow test.xml' for running a workflow test case as defined by supplied workflow xml test file (experimental) +'${0##*/} -framework' for running through example tool tests testing framework features in test/functional/tools" +'${0##*/} -framework -id toolid' for testing one framework tool (in test/functional/tools/) with id 'toolid' +'${0##*/} -data_managers -id data_manager_id' for testing one Data Manager with id 'data_manager_id' +EOF +} + +show_list() { + python tool_list.py + echo "===========================================================================================================================================" + echo "'${0##*/} -id bbb' for testing one tool with id 'bbb' ('bbb' is the tool id)" + echo "'${0##*/} -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" +} + +test_script="./scripts/functional_tests.py" +report_file="run_functional_tests.html" + +while : +do + case "$1" in + -h|--help|-\?) + show_help + exit 0 + ;; + -l|-list|--list) + show_list + exit 0 + ;; + -id|--id) + if [ $# -gt 1 ]; then + test_id=$2; + shift 2 + else + echo "--id requires an argument" 1>&2 + exit 1 + fi + ;; + -s|-sid|--sid) + if [ $# -gt 1 ]; then + section_id=$2 + shift 2 + else + echo "--sid requires an argument" 1>&2 + exit 1 + fi + ;; + -t|-toolshed|--toolshed) + test_script="./test/tool_shed/functional_tests.py" + report_file="./test/tool_shed/run_functional_tests.html" + if [ $# -gt 1 ]; then + toolshed_script=$2 + shift 2 + else + toolshed_script="./test/tool_shed/functional" + shift 1 + fi + ;; + -w|-workflow|--workflow) + if [ $# -gt 1 ]; then + workflow_file=$2 + workflow_test=1 + shift 2 + else + echo "--workflow requires an argument" 1>&2 + exit 1 + fi + ;; + -f|-framework|--framework) + framework_test=1; + shift 1 + ;; + -d|-data_managers|--data_managers) + data_managers_test=1; + shift 1 + ;; + -m|-migrated|--migrated) + migrated_test=1; + shift + ;; + -i|-installed|--installed) + installed_test=1; + shift + ;; + -r|--report_file) + if [ $# -gt 1 ]; then + report_file=$2 + shift 2 + else + echo "--report_file requires an argument" 1>&2 + exit 1 + fi + ;; + --) + shift + break + ;; + -*) + echo "invalid option: $1" 1>&2; + show_help + exit 1 + ;; + *) + break; + ;; + esac +done + +if [ -n "$migrated_test" ] ; then + [ -n "$test_id" ] && class=":TestForTool_$test_id" || class="" + extra_args="functional.test_toolbox$class -migrated" +elif [ -n "$installed_test" ] ; then + [ -n "$test_id" ] && class=":TestForTool_$test_id" || class="" + extra_args="functional.test_toolbox$class -installed" +elif [ -n "$framework_test" ] ; then + [ -n "$test_id" ] && class=":TestForTool_$test_id" || class="" + extra_args="functional.test_toolbox$class -framework" +elif [ -n "$data_managers_test" ] ; then + [ -n "$test_id" ] && class=":TestForDataManagerTool_$test_id" || class="" + extra_args="functional.test_data_managers$class -data_managers" +elif [ -n "$workflow_test" ]; then + extra_args="functional.workflow:WorkflowTestCase $workflow_file" +elif [ -n "$toolshed_script" ]; then + extra_args="$toolshed_script" +elif [ -n "$section_id" ]; then + extra_args=`python tool_list.py $section_id` +elif [ -n "$test_id" ]; then + class=":TestForTool_$test_id" + extra_args="functional.test_toolbox$class" +elif [ -n "$1" ] ; then + extra_args="$1" +else + extra_args='--exclude="^get" functional' +fi + +python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $extra_args https://bitbucket.org/galaxy/galaxy-central/commits/08754cf82e62/ Changeset: 08754cf82e62 User: jmchilton Date: 2014-04-28 18:48:51 Summary: Add unit tests to run_tests.sh. Affected #: 1 file diff -r 9dd03dd46d04b83aeb270ebe538d6075facb5379 -r 08754cf82e625cb6c6fa88aeb6e4195b222da64b run_tests.sh --- a/run_tests.sh +++ b/run_tests.sh @@ -16,6 +16,8 @@ '${0##*/} -framework' for running through example tool tests testing framework features in test/functional/tools" '${0##*/} -framework -id toolid' for testing one framework tool (in test/functional/tools/) with id 'toolid' '${0##*/} -data_managers -id data_manager_id' for testing one Data Manager with id 'data_manager_id' +'${0##*/} -unit' for running all unit tests (doctests in lib and tests in test/unit) +'${0##*/} -unit testscriptath' running particular tests scripts EOF } @@ -104,6 +106,24 @@ exit 1 fi ;; + -c|--coverage) + # Must have coverage installed (try `which coverage`) - only valid with --unit + # for now. Would be great to get this to work with functional tests though. + coverage_arg="--with-coverage" + NOSE_WITH_COVERAGE=true + shift + ;; + -u|-unit|--unit) + report_file="run_unit_tests.html" + test_script="./scripts/nosetests.py" + if [ $# -gt 1 ]; then + unit_extra=$2 + shift 2 + else + unit_extra='--exclude=functional --exclude="^get" --exclude=controllers --exclude=runners lib test/unit' + shift 1 + fi + ;; --) shift break @@ -140,6 +160,8 @@ elif [ -n "$test_id" ]; then class=":TestForTool_$test_id" extra_args="functional.test_toolbox$class" +elif [ -n "$unit_extra" ]; then + extra_args="--with-doctest $unit_extra" elif [ -n "$1" ] ; then extra_args="$1" else https://bitbucket.org/galaxy/galaxy-central/commits/e8047b8f1e24/ Changeset: e8047b8f1e24 User: jmchilton Date: 2014-04-28 18:48:51 Summary: Add qunit test options to new run_tests.sh script. Can target specific tests or all tests and can optionally watch tests using Carl's work on 'grunt watch'. Affected #: 1 file diff -r 08754cf82e625cb6c6fa88aeb6e4195b222da64b -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 run_tests.sh --- a/run_tests.sh +++ b/run_tests.sh @@ -18,6 +18,8 @@ '${0##*/} -data_managers -id data_manager_id' for testing one Data Manager with id 'data_manager_id' '${0##*/} -unit' for running all unit tests (doctests in lib and tests in test/unit) '${0##*/} -unit testscriptath' running particular tests scripts +'${0##*/} -qunit' for running qunit JavaScript tests +'${0##*/} -qunit testname' for running single JavaScript test with given name EOF } @@ -31,6 +33,8 @@ test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" +driver="python" + while : do case "$1" in @@ -124,6 +128,24 @@ shift 1 fi ;; + -q|-qunit|--qunit) + # Requires grunt installed and dependencies configured see + # test/qunit/README.txt for more information. + driver="grunt" + gruntfile="./test/qunit/Gruntfile.js" + if [ $# -gt 1 ]; then + qunit_name=$2 + shift 2 + else + shift 1 + fi + ;; + -watch|--watch) + # Have grunt watch test or directory for changes, only + # valid for javascript testing. + watch=1 + shift + ;; --) shift break @@ -168,4 +190,21 @@ extra_args='--exclude="^get" functional' fi -python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $extra_args +if [ "$driver" = "python" ]; then + python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $extra_args +else + if [ -n "$watch" ]; then + grunt_task="watch" + else + grunt_task="" + fi + if [ -n "$qunit_name" ]; then + grunt_args="--test=$qunit_name" + else + grunt_args="" + fi + # TODO: Exapnd javascript helpers to include setting up + # grunt deps in npm, "watch"ing directory, and running casper + # functional tests. + grunt --gruntfile=$gruntfile $grunt_task $grunt_args +fi https://bitbucket.org/galaxy/galaxy-central/commits/28b307e0c087/ Changeset: 28b307e0c087 User: jmchilton Date: 2014-04-28 18:48:51 Summary: Rework test framework testing and tools. -framework will still cause just a small set of non-default, non-user facing tools to be tested and exerise various aspects of the tool and test frameworks, but now an argument can be passed in -with_framework_test_tools to allow these tools to be used as part of the normal test framework (i.e. when not using -framework). This serves two purposes - to ensure the feature coverage of existing tests remains high as the tests are migrated out of the distribution and to allow a set of tests to be available for functional (e.g. API tests) that users don't need to see. Affected #: 15 files diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d run_tests.sh --- a/run_tests.sh +++ b/run_tests.sh @@ -32,6 +32,7 @@ test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" +with_framework_test_tools_arg="" driver="python" @@ -75,6 +76,10 @@ shift 1 fi ;; + -with_framework_test_tools|--with_framework_test_tools) + with_framework_test_tools_arg="-with_framework_test_tools" + shift + ;; -w|-workflow|--workflow) if [ $# -gt 1 ]; then workflow_file=$2 @@ -191,7 +196,7 @@ fi if [ "$driver" = "python" ]; then - python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $extra_args + python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $with_framework_test_tools_arg $extra_args else if [ -n "$watch" ]; then grunt_task="watch" diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -217,16 +217,19 @@ # Exclude all files except test_toolbox.py. ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) ) else: + framework_tool_dir = os.path.join('test', 'functional', 'tools') framework_test = __check_arg( '-framework' ) # Run through suite of tests testing framework. if framework_test: - framework_tool_dir = os.path.join('test', 'functional', 'tools') tool_conf = os.path.join( framework_tool_dir, 'samples_tool_conf.xml' ) datatypes_conf_override = os.path.join( framework_tool_dir, 'sample_datatypes_conf.xml' ) - test_dir = os.path.join( framework_tool_dir, 'test-data') else: # Use tool_conf.xml toolbox. tool_conf = 'tool_conf.xml' - test_dir = default_galaxy_test_file_dir + if __check_arg( '-with_framework_test_tools' ): + # Some of these tools will not work without swapping + # default interactor to point to test. + tool_conf = "%s,%s" % ( tool_conf, os.path.join( framework_tool_dir, 'samples_tool_conf.xml' ) ) + test_dir = default_galaxy_test_file_dir tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', tool_conf ) galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', test_dir ) if not os.path.isabs( galaxy_test_file_dir ): diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test-data/simple_line.txt --- /dev/null +++ b/test-data/simple_line.txt @@ -0,0 +1,1 @@ +This is a line of text. diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test-data/simple_line_alternative.txt --- /dev/null +++ b/test-data/simple_line_alternative.txt @@ -0,0 +1,1 @@ +This is a different line of text. \ No newline at end of file diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test-data/simple_line_x2.txt --- /dev/null +++ b/test-data/simple_line_x2.txt @@ -0,0 +1,2 @@ +This is a line of text. +This is a line of text. diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test-data/simple_line_x3.txt --- /dev/null +++ b/test-data/simple_line_x3.txt @@ -0,0 +1,3 @@ +This is a line of text. +This is a line of text. +This is a line of text. diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test-data/simple_line_x5.txt --- /dev/null +++ b/test-data/simple_line_x5.txt @@ -0,0 +1,5 @@ +This is a line of text. +This is a line of text. +This is a line of text. +This is a line of text. +This is a line of text. diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test-data/simple_lines_interleaved.txt --- /dev/null +++ b/test-data/simple_lines_interleaved.txt @@ -0,0 +1,4 @@ +This is a line of text. +This is a different line of text. +This is a line of text. +This is a different line of text. \ No newline at end of file diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test/functional/tools/test-data/simple_line.txt --- a/test/functional/tools/test-data/simple_line.txt +++ /dev/null @@ -1,1 +0,0 @@ -This is a line of text. diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test/functional/tools/test-data/simple_line_alternative.txt --- a/test/functional/tools/test-data/simple_line_alternative.txt +++ /dev/null @@ -1,1 +0,0 @@ -This is a different line of text. \ No newline at end of file diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test/functional/tools/test-data/simple_line_x2.txt --- a/test/functional/tools/test-data/simple_line_x2.txt +++ /dev/null @@ -1,2 +0,0 @@ -This is a line of text. -This is a line of text. diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test/functional/tools/test-data/simple_line_x3.txt --- a/test/functional/tools/test-data/simple_line_x3.txt +++ /dev/null @@ -1,3 +0,0 @@ -This is a line of text. -This is a line of text. -This is a line of text. diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test/functional/tools/test-data/simple_line_x5.txt --- a/test/functional/tools/test-data/simple_line_x5.txt +++ /dev/null @@ -1,5 +0,0 @@ -This is a line of text. -This is a line of text. -This is a line of text. -This is a line of text. -This is a line of text. diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test/functional/tools/test-data/simple_lines_interleaved.txt --- a/test/functional/tools/test-data/simple_lines_interleaved.txt +++ /dev/null @@ -1,4 +0,0 @@ -This is a line of text. -This is a different line of text. -This is a line of text. -This is a different line of text. \ No newline at end of file diff -r e8047b8f1e24a4815138e5862f24096dfc4d9c79 -r 28b307e0c08791b3073dd27c0189fdd28083045d test/functional/tools/test-data/velveth_test1 --- a/test/functional/tools/test-data/velveth_test1 +++ /dev/null @@ -1,1 +0,0 @@ -../../../../test-data/velveth_test1 \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: davebgx: Twill test framework cleanup. Update install and test methods to reflect changes in base twilltestcase.py.
by commits-noreply＠bitbucket.org 28 Apr '14

28 Apr '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4a6cd1da30d0/ Changeset: 4a6cd1da30d0 User: davebgx Date: 2014-04-28 17:22:17 Summary: Twill test framework cleanup. Update install and test methods to reflect changes in base twilltestcase.py. Affected #: 2 files diff -r fe0df2e688999aa7e0087cbb95c0a7257e9ae6d2 -r 4a6cd1da30d0c4e3f8e0ed7b1c1e6342bed20e44 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1769,10 +1769,11 @@ tc.submit( "edit_external_service_button" ) for check_str in strings_displayed_after_submit: self.check_page_for_string( check_str ) + # Sample tracking stuff def check_request_grid( self, cntrller, state, deleted=False, strings_displayed=[] ): - self.visit_url( '%s/%s/browse_requests?sort=create_time&f-state=%s&f-deleted=%s' % \ - ( self.url, cntrller, state.replace( ' ', '+' ), str( deleted ) ) ) + params = { 'f-state': state, 'f-deleted': deleted, 'sort': 'create_time' } + self.visit_url( '/%s/browse_requests' % cntrller ) for check_str in strings_displayed: self.check_page_for_string( check_str ) def create_request_type( self, name, desc, request_form_id, sample_form_id, states, strings_displayed=[], strings_displayed_after_submit=[] ): @@ -1967,8 +1968,8 @@ for check_str in strings_displayed: self.check_page_for_string( check_str ) for sample_id in sample_ids: - tc.fv( "1", "select_sample_%i" % sample_id, True ) - tc.fv( "1", "sample_operation", 'Select data library and folder' ) + tc.fv( "edit_samples", "select_sample_%i" % sample_id, True ) + tc.fv( "edit_samples", "sample_operation", 'Select data library and folder' ) # refresh on change to show the data libraries selectfield self.refresh_form( "sample_operation", 'Select data library and folder' ) self.check_page_for_string( "Select data library:" ) diff -r fe0df2e688999aa7e0087cbb95c0a7257e9ae6d2 -r 4a6cd1da30d0c4e3f8e0ed7b1c1e6342bed20e44 test/install_and_test_tool_shed_repositories/base/twilltestcase.py --- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py +++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py @@ -3,7 +3,6 @@ import re import test_db_util import time -import urllib import galaxy.model as model import galaxy.model.tool_shed_install as install_model @@ -72,22 +71,22 @@ encoded_repository_id = repository_info_dict[ 'repository_id' ] tool_shed_url = repository_info_dict[ 'tool_shed_url' ] # Pass galaxy_url to the tool shed in order to set cookies and redirects correctly. - install_params = urllib.urlencode( dict( repository_ids=encoded_repository_id, - changeset_revisions=changeset_revision, - galaxy_url=self.url ) ) + install_params = dict( repository_ids=encoded_repository_id, + changeset_revisions=changeset_revision, + galaxy_url=self.url ) # If the tool shed does not have the same hostname as the Galaxy server being used for these tests, # twill will not carry over previously set cookies for the Galaxy server when following the # install_repositories_by_revision redirect, so we have to include 403 in the allowed HTTP # status codes and log in again. - url = '%s/repository/install_repositories_by_revision?%s' % ( tool_shed_url, install_params ) - self.visit_url( url, allowed_codes=[ 200, 403 ] ) + url = '%s/repository/install_repositories_by_revision' % tool_shed_url + self.visit_url( url, params=install_params, allowed_codes=[ 200, 403 ] ) self.logout() self.login( email='test(a)bx.psu.edu', username='test' ) - install_params = urllib.urlencode( dict( repository_ids=encoded_repository_id, - changeset_revisions=changeset_revision, - tool_shed_url=tool_shed_url ) ) - url = '/admin_toolshed/prepare_for_install?%s' % install_params - self.visit_url( url ) + install_params = dict( repository_ids=encoded_repository_id, + changeset_revisions=changeset_revision, + tool_shed_url=tool_shed_url ) + url = '/admin_toolshed/prepare_for_install' + self.visit_url( url, params=install_params ) # This section is tricky, due to the way twill handles form submission. The tool dependency checkbox needs to # be hacked in through tc.browser, putting the form field in kwd doesn't work. form = tc.browser.get_form( 'select_tool_panel_section' ) @@ -124,13 +123,6 @@ del( kwd[ field_name ] ) return kwd - def visit_url( self, url, allowed_codes=[ 200 ] ): - new_url = tc.go( url ) - return_code = tc.browser.get_code() - assert return_code in allowed_codes, 'Invalid HTTP return code %s, allowed codes: %s' % \ - ( return_code, ', '.join( str( code ) for code in allowed_codes ) ) - return new_url - def wait_for_repository_installation( self, repository_ids ): final_states = [ install_model.ToolShedRepository.installation_status.ERROR, install_model.ToolShedRepository.installation_status.INSTALLED ] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Merged in jmchilton/galaxy-central-fork-1 (pull request #378)
by commits-noreply＠bitbucket.org 28 Apr '14

28 Apr '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4d554f1875eb/ Changeset: 4d554f1875eb User: jmchilton Date: 2014-04-28 16:53:07 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #378) Allow specification of environment variables for job destinations Affected #: 9 files diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -74,6 +74,23 @@ <param id="Resource_List">walltime=72:00:00</param></destination><destination id="remote_cluster" runner="drmaa" tags="longjobs"/> + <destination id="java_cluster" runner="drmaa"> +  + <env id="_JAVA_OPTIONS">-Xmx=6GB</env> + <env id="ANOTHER_OPTION" raw="true">'5'</env> + <env file="/mnt/java_cluster/environment_setup.sh" /> + <env exec="module load javastuff/2.10" /> +  + </destination><destination id="real_user_cluster" runner="drmaa"><param id="galaxy_external_runjob_script">scripts/drmaa_external_runner.py</param> diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -179,6 +179,7 @@ job_metrics.set_destination_conf_element( id, metrics_elements[ 0 ] ) job_destination = JobDestination(**dict(destination.items())) job_destination['params'] = self.__get_params(destination) + job_destination['env'] = self.__get_envs(destination) self.destinations[id] = (job_destination,) if job_destination.tags is not None: for tag in job_destination.tags: @@ -350,6 +351,25 @@ rval[param.get('id')] = param.text return rval + def __get_envs(self, parent): + """Parses any child <env> tags in to a dictionary suitable for persistence. + + :param parent: Parent element in which to find child <param> tags. + :type parent: ``xml.etree.ElementTree.Element`` + + :returns: dict + """ + rval = [] + for param in parent.findall('env'): + rval.append( dict( + name=param.get('id'), + file=param.get('file'), + execute=param.get('exec'), + value=param.text, + raw=util.asbool(param.get('raw', 'false')) + ) ) + return rval + @property def default_job_tool_configuration(self): """The default JobToolConfiguration, used if a tool does not have an explicit defintion in the configuration. It consists of a reference to the default handler and default destination. diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -17,6 +17,7 @@ from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size from galaxy.util import in_directory from galaxy.jobs.runners.util.job_script import job_script +from galaxy.jobs.runners.util.env import env_to_statement log = logging.getLogger( __name__ ) @@ -253,10 +254,17 @@ def get_job_file(self, job_wrapper, **kwds): job_metrics = job_wrapper.app.job_metrics job_instrumenter = job_metrics.job_instrumenters[ job_wrapper.job_destination.id ] + + env_setup_commands = kwds.get( 'env_setup_commands', [] ) + env_setup_commands.append( job_wrapper.get_env_setup_clause() or '' ) + destination = job_wrapper.job_destination or {} + envs = destination.get( "env", [] ) + for env in envs: + env_setup_commands.append( env_to_statement( env ) ) options = dict( job_instrumenter=job_instrumenter, galaxy_lib=job_wrapper.galaxy_lib_dir, - env_setup_commands=job_wrapper.get_env_setup_clause(), + env_setup_commands=env_setup_commands, working_directory=os.path.abspath( job_wrapper.working_directory ), command=job_wrapper.runner_command_line, ) diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -134,6 +134,7 @@ tool=job_wrapper.tool, config_files=job_wrapper.extra_filenames, requirements=requirements, + env=client.env, rewrite_paths=rewrite_paths, arbitrary_files=unstructured_path_rewrites, ) @@ -222,14 +223,15 @@ for key, value in params.iteritems(): if value: params[key] = model.User.expand_user_properties( job_wrapper.get_job().user, value ) - return self.get_client( params, job_id ) + env = getattr( job_wrapper.job_destination, "env", [] ) + return self.get_client( params, job_id, env ) def get_client_from_state(self, job_state): job_destination_params = job_state.job_destination.params job_id = job_state.job_id return self.get_client( job_destination_params, job_id ) - def get_client( self, job_destination_params, job_id ): + def get_client( self, job_destination_params, job_id, env=[] ): # Cannot use url_for outside of web thread. #files_endpoint = url_for( controller="job_files", job_id=encoded_job_id ) @@ -243,6 +245,7 @@ get_client_kwds = dict( job_id=str( job_id ), files_endpoint=files_endpoint, + env=env ) return self.client_manager.get_client( job_destination_params, **get_client_kwds ) diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -41,6 +41,7 @@ job_directory = None self.env = destination_params.get( "env", [] ) self.files_endpoint = destination_params.get("files_endpoint", None) + self.env = destination_params.get("env", []) self.job_directory = job_directory self.default_file_action = self.destination_params.get("default_file_action", "transfer") diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -296,7 +296,7 @@ else: stage_commands = '' - env_setup_commands = '%s\n%s' % (stage_commands, job_wrapper.get_env_setup_clause()) + env_setup_commands = [ stage_commands ] script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands) job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) fh = file(job_file, "w") diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/util/env.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/env.py @@ -0,0 +1,40 @@ + +RAW_VALUE_BY_DEFAULT = False + + +def env_to_statement(env): + ''' Return the abstraction description of an environment variable definition + into a statement for shell script. + + >>> env_to_statement(dict(name='X', value='Y')) + 'X="Y"; export X' + >>> env_to_statement(dict(name='X', value='Y', raw=True)) + 'X=Y; export X' + >>> env_to_statement(dict(name='X', value='"A","B","C"')) + 'X="\\\\"A\\\\",\\\\"B\\\\",\\\\"C\\\\""; export X' + >>> env_to_statement(dict(file="Y")) + '. "Y"' + >>> env_to_statement(dict(file="'RAW $FILE'", raw=True)) + ". 'RAW $FILE'" + >>> # Source file takes precedence + >>> env_to_statement(dict(name='X', value='"A","B","C"', file="S")) + '. "S"' + >>> env_to_statement(dict(execute="module load java/1.5.1")) + 'module load java/1.5.1' + ''' + source_file = env.get('file', None) + if source_file: + return '. %s' % __escape(source_file, env) + execute = env.get('execute', None) + if execute: + return execute + name = env['name'] + value = __escape(env['value'], env) + return '%s=%s; export %s' % (name, value, name) + + +def __escape(value, env): + raw = env.get('raw', RAW_VALUE_BY_DEFAULT) + if not raw: + value = '"' + value.replace('"', '\\"') + '"' + return value diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/util/job_script/__init__.py --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -16,7 +16,7 @@ OPTIONAL_TEMPLATE_PARAMS = { 'galaxy_lib': None, 'headers': '', - 'env_setup_commands': '', + 'env_setup_commands': [], 'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT, 'instrument_pre_commands': '', 'instrument_post_commands': '', @@ -51,13 +51,15 @@ raise Exception("Failed to create job_script, a required parameter is missing.") job_instrumenter = kwds.get("job_instrumenter", None) if job_instrumenter: - del kwds[ "job_instrumenter" ] + del kwds["job_instrumenter"] working_directory = kwds["working_directory"] kwds["instrument_pre_commands"] = job_instrumenter.pre_execute_commands(working_directory) or '' kwds["instrument_post_commands"] = job_instrumenter.post_execute_commands(working_directory) or '' template_params = OPTIONAL_TEMPLATE_PARAMS.copy() template_params.update(**kwds) + env_setup_commands_str = "\n".join(template_params["env_setup_commands"]) + template_params["env_setup_commands"] = env_setup_commands_str if not isinstance(template, Template): template = Template(template) return template.safe_substitute(template_params) diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee test/unit/jobs/test_job_configuration.py --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -111,6 +111,20 @@ assert limits.concurrent_jobs[ "longjobs" ] == 1 assert limits.walltime_delta == datetime.timedelta( 0, 0, 0, 0, 0, 24 ) + def test_env_parsing( self ): + self.__with_advanced_config() + env_dest = self.job_config.destinations[ "java_cluster" ][ 0 ] + assert len( env_dest.env ) == 4, len( env_dest.env ) + assert env_dest.env[ 0 ][ "name" ] == "_JAVA_OPTIONS" + assert env_dest.env[ 0 ][ "value" ] == '-Xmx=6GB' + + assert env_dest.env[ 1 ][ "name" ] == "ANOTHER_OPTION" + assert env_dest.env[ 1 ][ "raw" ] is True + + assert env_dest.env[ 2 ][ "file" ] == "/mnt/java_cluster/environment_setup.sh" + + assert env_dest.env[ 3 ][ "execute" ] == "module load javastuff/2.10" + # TODO: Add job metrics parsing test. @property Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 28 Apr '14

28 Apr '14

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d48a5868cd1d/ Changeset: d48a5868cd1d User: jmchilton Date: 2014-04-24 07:36:54 Summary: Jobs - Convert job_script's env_setup_commands to list. Affected #: 3 files diff -r 641f7063ce3c5dcf375aafdd9a65a2e3f7ccfa4c -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -253,10 +253,13 @@ def get_job_file(self, job_wrapper, **kwds): job_metrics = job_wrapper.app.job_metrics job_instrumenter = job_metrics.job_instrumenters[ job_wrapper.job_destination.id ] + + env_setup_commands = kwds.get( 'env_setup_commands', [] ) + env_setup_commands.append( job_wrapper.get_env_setup_clause() or '' ) options = dict( job_instrumenter=job_instrumenter, galaxy_lib=job_wrapper.galaxy_lib_dir, - env_setup_commands=job_wrapper.get_env_setup_clause(), + env_setup_commands=[job_wrapper.get_env_setup_clause()], working_directory=os.path.abspath( job_wrapper.working_directory ), command=job_wrapper.runner_command_line, ) diff -r 641f7063ce3c5dcf375aafdd9a65a2e3f7ccfa4c -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -296,7 +296,7 @@ else: stage_commands = '' - env_setup_commands = '%s\n%s' % (stage_commands, job_wrapper.get_env_setup_clause()) + env_setup_commands = [ stage_commands ] script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands) job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) fh = file(job_file, "w") diff -r 641f7063ce3c5dcf375aafdd9a65a2e3f7ccfa4c -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 lib/galaxy/jobs/runners/util/job_script/__init__.py --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -16,7 +16,7 @@ OPTIONAL_TEMPLATE_PARAMS = { 'galaxy_lib': None, 'headers': '', - 'env_setup_commands': '', + 'env_setup_commands': [], 'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT, 'instrument_pre_commands': '', 'instrument_post_commands': '', @@ -58,6 +58,8 @@ template_params = OPTIONAL_TEMPLATE_PARAMS.copy() template_params.update(**kwds) + env_setup_commands_str = "\n".join(template_params["env_setup_commands"]) + template_params["env_setup_commands"] = env_setup_commands_str if not isinstance(template, Template): template = Template(template) return template.safe_substitute(template_params) https://bitbucket.org/galaxy/galaxy-central/commits/63d6d9a9af79/ Changeset: 63d6d9a9af79 User: jmchilton Date: 2014-04-24 07:36:54 Summary: Jobs - Allow destinations to declare arbitrary envirnoment variables. See job_conf.xml.sample_advanced for examples. Affected #: 8 files diff -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -74,6 +74,13 @@ <param id="Resource_List">walltime=72:00:00</param></destination><destination id="remote_cluster" runner="drmaa" tags="longjobs"/> + <destination id="java_cluster" runner="drmaa"> +  + <env id="_JAVA_OPTIONS">-Xmx=6GB</env> + <env id="ANOTHER_OPTION" raw="true">'5'</env> + </destination><destination id="real_user_cluster" runner="drmaa"><param id="galaxy_external_runjob_script">scripts/drmaa_external_runner.py</param> diff -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -179,6 +179,7 @@ job_metrics.set_destination_conf_element( id, metrics_elements[ 0 ] ) job_destination = JobDestination(**dict(destination.items())) job_destination['params'] = self.__get_params(destination) + job_destination['env'] = self.__get_envs(destination) self.destinations[id] = (job_destination,) if job_destination.tags is not None: for tag in job_destination.tags: @@ -350,6 +351,23 @@ rval[param.get('id')] = param.text return rval + def __get_envs(self, parent): + """Parses any child <env> tags in to a dictionary suitable for persistence. + + :param parent: Parent element in which to find child <param> tags. + :type parent: ``xml.etree.ElementTree.Element`` + + :returns: dict + """ + rval = [] + for param in parent.findall('env'): + rval.append( dict( + name=param.get('id'), + value=param.text, + raw=util.asbool(param.get('raw', 'false')) + ) ) + return rval + @property def default_job_tool_configuration(self): """The default JobToolConfiguration, used if a tool does not have an explicit defintion in the configuration. It consists of a reference to the default handler and default destination. diff -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -17,6 +17,7 @@ from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size from galaxy.util import in_directory from galaxy.jobs.runners.util.job_script import job_script +from galaxy.jobs.runners.util.env import env_to_statement log = logging.getLogger( __name__ ) @@ -256,10 +257,14 @@ env_setup_commands = kwds.get( 'env_setup_commands', [] ) env_setup_commands.append( job_wrapper.get_env_setup_clause() or '' ) + destination = job_wrapper.job_destination or {} + envs = destination.get( "env", [] ) + for env in envs: + env_setup_commands.append( env_to_statement( env ) ) options = dict( job_instrumenter=job_instrumenter, galaxy_lib=job_wrapper.galaxy_lib_dir, - env_setup_commands=[job_wrapper.get_env_setup_clause()], + env_setup_commands=env_setup_commands, working_directory=os.path.abspath( job_wrapper.working_directory ), command=job_wrapper.runner_command_line, ) diff -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -134,6 +134,7 @@ tool=job_wrapper.tool, config_files=job_wrapper.extra_filenames, requirements=requirements, + env=client.env, rewrite_paths=rewrite_paths, arbitrary_files=unstructured_path_rewrites, ) @@ -222,14 +223,15 @@ for key, value in params.iteritems(): if value: params[key] = model.User.expand_user_properties( job_wrapper.get_job().user, value ) - return self.get_client( params, job_id ) + env = getattr( job_wrapper.job_destination, "env", [] ) + return self.get_client( params, job_id, env ) def get_client_from_state(self, job_state): job_destination_params = job_state.job_destination.params job_id = job_state.job_id return self.get_client( job_destination_params, job_id ) - def get_client( self, job_destination_params, job_id ): + def get_client( self, job_destination_params, job_id, env=[] ): # Cannot use url_for outside of web thread. #files_endpoint = url_for( controller="job_files", job_id=encoded_job_id ) @@ -243,6 +245,7 @@ get_client_kwds = dict( job_id=str( job_id ), files_endpoint=files_endpoint, + env=env ) return self.client_manager.get_client( job_destination_params, **get_client_kwds ) diff -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -41,6 +41,7 @@ job_directory = None self.env = destination_params.get( "env", [] ) self.files_endpoint = destination_params.get("files_endpoint", None) + self.env = destination_params.get("env", []) self.job_directory = job_directory self.default_file_action = self.destination_params.get("default_file_action", "transfer") diff -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 lib/galaxy/jobs/runners/util/env.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/env.py @@ -0,0 +1,21 @@ + +RAW_VALUE_BY_DEFAULT = False + + +def env_to_statement(env): + ''' Return the abstraction description of an environment variable definition + into a statement for shell script. + + >>> env_to_statement(dict(name='X', value='Y')) + 'X="Y"; export X' + >>> env_to_statement(dict(name='X', value='Y', raw=True)) + 'X=Y; export X' + >>> env_to_statement(dict(name='X', value='"A","B","C"')) + 'X="\\\\"A\\\\",\\\\"B\\\\",\\\\"C\\\\""; export X' + ''' + name = env['name'] + value = env['value'] + raw = env.get('raw', RAW_VALUE_BY_DEFAULT) + if not raw: + value = '"' + value.replace('"', '\\"') + '"' + return '%s=%s; export %s' % (name, value, name) diff -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 lib/galaxy/jobs/runners/util/job_script/__init__.py --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -51,7 +51,7 @@ raise Exception("Failed to create job_script, a required parameter is missing.") job_instrumenter = kwds.get("job_instrumenter", None) if job_instrumenter: - del kwds[ "job_instrumenter" ] + del kwds["job_instrumenter"] working_directory = kwds["working_directory"] kwds["instrument_pre_commands"] = job_instrumenter.pre_execute_commands(working_directory) or '' kwds["instrument_post_commands"] = job_instrumenter.post_execute_commands(working_directory) or '' diff -r d48a5868cd1dc986b1f9ea2b0b8931407e124733 -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 test/unit/jobs/test_job_configuration.py --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -111,6 +111,16 @@ assert limits.concurrent_jobs[ "longjobs" ] == 1 assert limits.walltime_delta == datetime.timedelta( 0, 0, 0, 0, 0, 24 ) + def test_env_parsing( self ): + self.__with_advanced_config() + env_dest = self.job_config.destinations[ "java_cluster" ][ 0 ] + assert len( env_dest.env ) == 2, len( env_dest.env ) + assert env_dest.env[ 0 ][ "name" ] == "_JAVA_OPTIONS" + assert env_dest.env[ 0 ][ "value" ] == '-Xmx=6GB' + + assert env_dest.env[ 1 ][ "name" ] == "ANOTHER_OPTION" + assert env_dest.env[ 1 ][ "raw" ] is True + # TODO: Add job metrics parsing test. @property https://bitbucket.org/galaxy/galaxy-central/commits/d72967b2b7b2/ Changeset: d72967b2b7b2 User: jmchilton Date: 2014-04-25 16:36:40 Summary: Extend destination specific env options. Add support sourcing files and executing arbitrary shell commands (see documented examples). Affected #: 4 files diff -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 -r d72967b2b7b25ba02584880526c7b28cab6970e7 job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -77,9 +77,19 @@ <destination id="java_cluster" runner="drmaa"> + DRMAA/SLURM, PBS, Condor, and LWR. General dependencies for tools + should be configured via tool_depednency_dir and package options + and these options should be reserved for defining cluster specific + options. + --><env id="_JAVA_OPTIONS">-Xmx=6GB</env><env id="ANOTHER_OPTION" raw="true">'5'</env> + <env file="/mnt/java_cluster/environment_setup.sh" /> + <env exec="module load javastuff/2.10" /> + </destination><destination id="real_user_cluster" runner="drmaa"> diff -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 -r d72967b2b7b25ba02584880526c7b28cab6970e7 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -363,6 +363,8 @@ for param in parent.findall('env'): rval.append( dict( name=param.get('id'), + file=param.get('file'), + execute=param.get('exec'), value=param.text, raw=util.asbool(param.get('raw', 'false')) ) ) diff -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 -r d72967b2b7b25ba02584880526c7b28cab6970e7 lib/galaxy/jobs/runners/util/env.py --- a/lib/galaxy/jobs/runners/util/env.py +++ b/lib/galaxy/jobs/runners/util/env.py @@ -12,10 +12,29 @@ 'X=Y; export X' >>> env_to_statement(dict(name='X', value='"A","B","C"')) 'X="\\\\"A\\\\",\\\\"B\\\\",\\\\"C\\\\""; export X' + >>> env_to_statement(dict(file="Y")) + '. "Y"' + >>> env_to_statement(dict(file="'RAW $FILE'", raw=True)) + ". 'RAW $FILE'" + >>> # Source file takes precedence + >>> env_to_statement(dict(name='X', value='"A","B","C"', file="S")) + '. "S"' + >>> env_to_statement(dict(execute="module load java/1.5.1")) + 'module load java/1.5.1' ''' + source_file = env.get('file', None) + if source_file: + return '. %s' % __escape(source_file, env) + execute = env.get('execute', None) + if execute: + return execute name = env['name'] - value = env['value'] + value = __escape(env['value'], env) + return '%s=%s; export %s' % (name, value, name) + + +def __escape(value, env): raw = env.get('raw', RAW_VALUE_BY_DEFAULT) if not raw: value = '"' + value.replace('"', '\\"') + '"' - return '%s=%s; export %s' % (name, value, name) + return value diff -r 63d6d9a9af796d66f44e5105d306e3c46ff09549 -r d72967b2b7b25ba02584880526c7b28cab6970e7 test/unit/jobs/test_job_configuration.py --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -114,13 +114,17 @@ def test_env_parsing( self ): self.__with_advanced_config() env_dest = self.job_config.destinations[ "java_cluster" ][ 0 ] - assert len( env_dest.env ) == 2, len( env_dest.env ) + assert len( env_dest.env ) == 4, len( env_dest.env ) assert env_dest.env[ 0 ][ "name" ] == "_JAVA_OPTIONS" assert env_dest.env[ 0 ][ "value" ] == '-Xmx=6GB' assert env_dest.env[ 1 ][ "name" ] == "ANOTHER_OPTION" assert env_dest.env[ 1 ][ "raw" ] is True + assert env_dest.env[ 2 ][ "file" ] == "/mnt/java_cluster/environment_setup.sh" + + assert env_dest.env[ 3 ][ "execute" ] == "module load javastuff/2.10" + # TODO: Add job metrics parsing test. @property https://bitbucket.org/galaxy/galaxy-central/commits/4d554f1875eb/ Changeset: 4d554f1875eb User: jmchilton Date: 2014-04-28 16:53:07 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #378) Allow specification of environment variables for job destinations Affected #: 9 files diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -74,6 +74,23 @@ <param id="Resource_List">walltime=72:00:00</param></destination><destination id="remote_cluster" runner="drmaa" tags="longjobs"/> + <destination id="java_cluster" runner="drmaa"> +  + <env id="_JAVA_OPTIONS">-Xmx=6GB</env> + <env id="ANOTHER_OPTION" raw="true">'5'</env> + <env file="/mnt/java_cluster/environment_setup.sh" /> + <env exec="module load javastuff/2.10" /> +  + </destination><destination id="real_user_cluster" runner="drmaa"><param id="galaxy_external_runjob_script">scripts/drmaa_external_runner.py</param> diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -179,6 +179,7 @@ job_metrics.set_destination_conf_element( id, metrics_elements[ 0 ] ) job_destination = JobDestination(**dict(destination.items())) job_destination['params'] = self.__get_params(destination) + job_destination['env'] = self.__get_envs(destination) self.destinations[id] = (job_destination,) if job_destination.tags is not None: for tag in job_destination.tags: @@ -350,6 +351,25 @@ rval[param.get('id')] = param.text return rval + def __get_envs(self, parent): + """Parses any child <env> tags in to a dictionary suitable for persistence. + + :param parent: Parent element in which to find child <param> tags. + :type parent: ``xml.etree.ElementTree.Element`` + + :returns: dict + """ + rval = [] + for param in parent.findall('env'): + rval.append( dict( + name=param.get('id'), + file=param.get('file'), + execute=param.get('exec'), + value=param.text, + raw=util.asbool(param.get('raw', 'false')) + ) ) + return rval + @property def default_job_tool_configuration(self): """The default JobToolConfiguration, used if a tool does not have an explicit defintion in the configuration. It consists of a reference to the default handler and default destination. diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -17,6 +17,7 @@ from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size from galaxy.util import in_directory from galaxy.jobs.runners.util.job_script import job_script +from galaxy.jobs.runners.util.env import env_to_statement log = logging.getLogger( __name__ ) @@ -253,10 +254,17 @@ def get_job_file(self, job_wrapper, **kwds): job_metrics = job_wrapper.app.job_metrics job_instrumenter = job_metrics.job_instrumenters[ job_wrapper.job_destination.id ] + + env_setup_commands = kwds.get( 'env_setup_commands', [] ) + env_setup_commands.append( job_wrapper.get_env_setup_clause() or '' ) + destination = job_wrapper.job_destination or {} + envs = destination.get( "env", [] ) + for env in envs: + env_setup_commands.append( env_to_statement( env ) ) options = dict( job_instrumenter=job_instrumenter, galaxy_lib=job_wrapper.galaxy_lib_dir, - env_setup_commands=job_wrapper.get_env_setup_clause(), + env_setup_commands=env_setup_commands, working_directory=os.path.abspath( job_wrapper.working_directory ), command=job_wrapper.runner_command_line, ) diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -134,6 +134,7 @@ tool=job_wrapper.tool, config_files=job_wrapper.extra_filenames, requirements=requirements, + env=client.env, rewrite_paths=rewrite_paths, arbitrary_files=unstructured_path_rewrites, ) @@ -222,14 +223,15 @@ for key, value in params.iteritems(): if value: params[key] = model.User.expand_user_properties( job_wrapper.get_job().user, value ) - return self.get_client( params, job_id ) + env = getattr( job_wrapper.job_destination, "env", [] ) + return self.get_client( params, job_id, env ) def get_client_from_state(self, job_state): job_destination_params = job_state.job_destination.params job_id = job_state.job_id return self.get_client( job_destination_params, job_id ) - def get_client( self, job_destination_params, job_id ): + def get_client( self, job_destination_params, job_id, env=[] ): # Cannot use url_for outside of web thread. #files_endpoint = url_for( controller="job_files", job_id=encoded_job_id ) @@ -243,6 +245,7 @@ get_client_kwds = dict( job_id=str( job_id ), files_endpoint=files_endpoint, + env=env ) return self.client_manager.get_client( job_destination_params, **get_client_kwds ) diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -41,6 +41,7 @@ job_directory = None self.env = destination_params.get( "env", [] ) self.files_endpoint = destination_params.get("files_endpoint", None) + self.env = destination_params.get("env", []) self.job_directory = job_directory self.default_file_action = self.destination_params.get("default_file_action", "transfer") diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -296,7 +296,7 @@ else: stage_commands = '' - env_setup_commands = '%s\n%s' % (stage_commands, job_wrapper.get_env_setup_clause()) + env_setup_commands = [ stage_commands ] script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands) job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) fh = file(job_file, "w") diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/util/env.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/env.py @@ -0,0 +1,40 @@ + +RAW_VALUE_BY_DEFAULT = False + + +def env_to_statement(env): + ''' Return the abstraction description of an environment variable definition + into a statement for shell script. + + >>> env_to_statement(dict(name='X', value='Y')) + 'X="Y"; export X' + >>> env_to_statement(dict(name='X', value='Y', raw=True)) + 'X=Y; export X' + >>> env_to_statement(dict(name='X', value='"A","B","C"')) + 'X="\\\\"A\\\\",\\\\"B\\\\",\\\\"C\\\\""; export X' + >>> env_to_statement(dict(file="Y")) + '. "Y"' + >>> env_to_statement(dict(file="'RAW $FILE'", raw=True)) + ". 'RAW $FILE'" + >>> # Source file takes precedence + >>> env_to_statement(dict(name='X', value='"A","B","C"', file="S")) + '. "S"' + >>> env_to_statement(dict(execute="module load java/1.5.1")) + 'module load java/1.5.1' + ''' + source_file = env.get('file', None) + if source_file: + return '. %s' % __escape(source_file, env) + execute = env.get('execute', None) + if execute: + return execute + name = env['name'] + value = __escape(env['value'], env) + return '%s=%s; export %s' % (name, value, name) + + +def __escape(value, env): + raw = env.get('raw', RAW_VALUE_BY_DEFAULT) + if not raw: + value = '"' + value.replace('"', '\\"') + '"' + return value diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee lib/galaxy/jobs/runners/util/job_script/__init__.py --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -16,7 +16,7 @@ OPTIONAL_TEMPLATE_PARAMS = { 'galaxy_lib': None, 'headers': '', - 'env_setup_commands': '', + 'env_setup_commands': [], 'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT, 'instrument_pre_commands': '', 'instrument_post_commands': '', @@ -51,13 +51,15 @@ raise Exception("Failed to create job_script, a required parameter is missing.") job_instrumenter = kwds.get("job_instrumenter", None) if job_instrumenter: - del kwds[ "job_instrumenter" ] + del kwds["job_instrumenter"] working_directory = kwds["working_directory"] kwds["instrument_pre_commands"] = job_instrumenter.pre_execute_commands(working_directory) or '' kwds["instrument_post_commands"] = job_instrumenter.post_execute_commands(working_directory) or '' template_params = OPTIONAL_TEMPLATE_PARAMS.copy() template_params.update(**kwds) + env_setup_commands_str = "\n".join(template_params["env_setup_commands"]) + template_params["env_setup_commands"] = env_setup_commands_str if not isinstance(template, Template): template = Template(template) return template.safe_substitute(template_params) diff -r b0067c6e06274e80157496e034c9c7d0a083ffb6 -r 4d554f1875eb9e1dfc5fc156dbae86c79c2efbee test/unit/jobs/test_job_configuration.py --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -111,6 +111,20 @@ assert limits.concurrent_jobs[ "longjobs" ] == 1 assert limits.walltime_delta == datetime.timedelta( 0, 0, 0, 0, 0, 24 ) + def test_env_parsing( self ): + self.__with_advanced_config() + env_dest = self.job_config.destinations[ "java_cluster" ][ 0 ] + assert len( env_dest.env ) == 4, len( env_dest.env ) + assert env_dest.env[ 0 ][ "name" ] == "_JAVA_OPTIONS" + assert env_dest.env[ 0 ][ "value" ] == '-Xmx=6GB' + + assert env_dest.env[ 1 ][ "name" ] == "ANOTHER_OPTION" + assert env_dest.env[ 1 ][ "raw" ] is True + + assert env_dest.env[ 2 ][ "file" ] == "/mnt/java_cluster/environment_setup.sh" + + assert env_dest.env[ 3 ][ "execute" ] == "module load javastuff/2.10" + # TODO: Add job metrics parsing test. @property Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Reorder eggs.require for some fabric files.
by commits-noreply＠bitbucket.org 28 Apr '14

28 Apr '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b0067c6e0627/ Changeset: b0067c6e0627 User: greg Date: 2014-04-28 16:37:44 Summary: Reorder eggs.require for some fabric files. Affected #: 3 files diff -r 9faa9a2ab1e544b8ba705b3e320d2166d7a6ee88 -r b0067c6e06274e80157496e034c9c7d0a083ffb6 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -3,9 +3,9 @@ from galaxy import eggs +eggs.require( 'paramiko' ) +eggs.require( 'ssh' ) eggs.require( 'Fabric' ) -eggs.require( 'ssh' ) -eggs.require( 'paramiko' ) from fabric.api import env from fabric.api import lcd diff -r 9faa9a2ab1e544b8ba705b3e320d2166d7a6ee88 -r b0067c6e06274e80157496e034c9c7d0a083ffb6 lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py @@ -12,9 +12,10 @@ # TODO: eliminate the use of fabric here. from galaxy import eggs + +eggs.require( 'paramiko' ) +eggs.require( 'ssh' ) eggs.require( 'Fabric' ) -eggs.require( 'ssh' ) -eggs.require( 'paramiko' ) from fabric.operations import _AttributeString from fabric import state diff -r 9faa9a2ab1e544b8ba705b3e320d2166d7a6ee88 -r b0067c6e06274e80157496e034c9c7d0a083ffb6 lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -13,9 +13,10 @@ # TODO: eliminate the use of fabric here. from galaxy import eggs + +eggs.require( 'paramiko' ) +eggs.require( 'ssh' ) eggs.require( 'Fabric' ) -eggs.require( 'ssh' ) -eggs.require( 'paramiko' ) from fabric.api import settings from fabric.api import lcd Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for the install_repository_revision function in the Galaxy API: a more correct and informative message is now displayed if any of the request parameters is invalid.
by commits-noreply＠bitbucket.org 28 Apr '14

28 Apr '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9faa9a2ab1e5/ Changeset: 9faa9a2ab1e5 User: greg Date: 2014-04-28 15:45:01 Summary: Fix for the install_repository_revision function in the Galaxy API: a more correct and informative message is now displayed if any of the request parameters is invalid. Affected #: 1 file diff -r dc9fb7fba975f6169d9ee8eb257bf477d7d4274c -r 9faa9a2ab1e544b8ba705b3e320d2166d7a6ee88 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -230,8 +230,18 @@ ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) ) log.debug( message ) return dict( status='error', error=message ) + # Make sure the tool shed returned everything we need for installing the repository. + if not repository_revision_dict or not repo_info_dict: + key = kwd.get( 'key', None ) + invalid_parameter_message = "No information is available for the requested repository revision.\n" + invalid_parameter_message += "One or more of the following parameter values is likely invalid:\n" + invalid_parameter_message += "key: %s\n" % str( key ) + invalid_parameter_message += "tool_shed_url: %s\n" % str( tool_shed_url ) + invalid_parameter_message += "name: %s\n" % str( name ) + invalid_parameter_message += "owner: %s\n" % str( owner ) + invalid_parameter_message += "changeset_revision: %s\n" % str( changeset_revision ) + raise HTTPBadRequest( detail=invalid_parameter_message ) repo_info_dicts = [ repo_info_dict ] - # Make sure the tool shed returned everything we need for installing the repository. try: has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] except: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: A bit of rework for recently introduced assert action types for tool dependency package install recipes: 1) assert_file_exists - true for both files and symlinks, but false if path is a directory 2) assert_file_executable - true for both files and symlinks but false if path is a directory 3) assert_directory_exists - true for both directories and symlinks, but false if path is a file 4) assert_directory_executable - true for both directories and symlinks, but fa
by commits-noreply＠bitbucket.org 28 Apr '14

28 Apr '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/dc9fb7fba975/ Changeset: dc9fb7fba975 User: greg Date: 2014-04-28 13:03:37 Summary: A bit of rework for recently introduced assert action types for tool dependency package install recipes: 1) assert_file_exists - true for both files and symlinks, but false if path is a directory 2) assert_file_executable - true for both files and symlinks but false if path is a directory 3) assert_directory_exists - true for both directories and symlinks, but false if path is a file 4) assert_directory_executable - true for both directories and symlinks, but false if path is a file. Affected #: 3 files diff -r 56dd83576ce67763863b61ad85104f8d14d978e0 -r dc9fb7fba975f6169d9ee8eb257bf477d7d4274c lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py @@ -414,7 +414,8 @@ return tool_dependency, filtered_actions, dir def load_step_handlers( self ): - step_handlers_by_type = dict( assert_directory_exists=step_handler.AssertDirectoryExists(), + step_handlers_by_type = dict( assert_directory_executable=step_handler.AssertDirectoryExecutable(), + assert_directory_exists=step_handler.AssertDirectoryExists(), assert_file_executable=step_handler.AssertFileExecutable(), assert_file_exists=step_handler.AssertFileExists(), autoconf=step_handler.Autoconf(), diff -r 56dd83576ce67763863b61ad85104f8d14d978e0 -r dc9fb7fba975f6169d9ee8eb257bf477d7d4274c lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -36,6 +36,39 @@ raise "Unimplemented Method" +class AssertDirectoryExecutable( RecipeStep ): + + def __init__( self ): + self.type = 'assert_directory_executable' + + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, + install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + """ + Make sure a symbolic link or directory on disk exists and is executable, but is not a file. + Since this class is not used in the initial download stage, no recipe step filtering is + performed here, and None values are always returned for filtered_actions and dir. + """ + if os.path.isabs( action_dict[ 'full_path' ] ): + full_path = action_dict[ 'full_path' ] + else: + full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) + if not td_common_util.assert_directory_executable( full_path=full_path ): + status = app.install_model.ToolDependency.installation_status.ERROR + error_message = 'The path %s is not a directory or is not executable by the owner.' % str( full_path ) + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency, + status=status, + error_message=error_message, + remove_from_disk=False ) + return tool_dependency, None, None + + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> + if action_elem.text: + action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_dir ) + return action_dict + + class AssertDirectoryExists( RecipeStep ): def __init__( self ): @@ -44,9 +77,9 @@ def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): """ - Make sure a directory on disk exists. Since this class is not used in the initial download stage, - no recipe step filtering is performed here, and None values are always returned for filtered_actions - and dir. + Make sure a a symbolic link or directory on disk exists, but is not a file. Since this + class is not used in the initial download stage, no recipe step filtering is performed + here, and None values are always returned for filtered_actions and dir. """ if os.path.isabs( action_dict[ 'full_path' ] ): full_path = action_dict[ 'full_path' ] @@ -54,7 +87,7 @@ full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) if not td_common_util.assert_directory_exists( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR - error_message = 'The required directory %s does not exist.' % str( full_path ) + error_message = 'The path %s is not a directory or does not exist.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, tool_dependency, status=status, @@ -77,9 +110,9 @@ def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): """ - Make sure a file on disk exists and is executable. Since this class is not used in the initial - download stage, no recipe step filtering is performed here, and None values are always returned - for filtered_actions and dir. + Make sure a symbolic link or file on disk exists and is executable, but is not a directory. + Since this class is not used in the initial download stage, no recipe step filtering is + performed here, and None values are always returned for filtered_actions and dir. """ if os.path.isabs( action_dict[ 'full_path' ] ): full_path = action_dict[ 'full_path' ] @@ -87,7 +120,7 @@ full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) if not td_common_util.assert_file_executable( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR - error_message = 'The file %s is not executable by the owner.' % str( full_path ) + error_message = 'The path %s is not a file or is not executable by the owner.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, tool_dependency, status=status, @@ -110,9 +143,9 @@ def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): """ - Make sure a file on disk exists. Since this class is not used in the initial download stage, - no recipe step filtering is performed here, and None values are always returned for - filtered_actions and dir. + Make sure a symbolic link or file on disk exists, but is not a directory. Since this + class is not used in the initial download stage, no recipe step filtering is performed + here, and None values are always returned for filtered_actions and dir. """ if os.path.isabs( action_dict[ 'full_path' ] ): full_path = action_dict[ 'full_path' ] @@ -120,7 +153,7 @@ full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) if not td_common_util.assert_file_exists( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR - error_message = 'The required file %s does not exist.' % str( full_path ) + error_message = 'The path %s is not a file or does not exist.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, tool_dependency, status=status, diff -r 56dd83576ce67763863b61ad85104f8d14d978e0 -r dc9fb7fba975f6169d9ee8eb257bf477d7d4274c lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -111,31 +111,63 @@ def open_zip( self, filepath, mode ): return zipfile.ZipFile( filepath, mode ) -def assert_directory_exists( full_path ): - """Return True if a directory exists and is not a symbolic link.""" - if os.path.islink( full_path ): +def assert_directory_executable( full_path ): + """ + Return True if a symbolic link or directory exists and is executable, but if + full_path is a file, return False. + """ + if full_path is None: return False - if os.path.is_dir( full_path ): - return True + if os.path.isfile( full_path ): + return False + if os.path.isdir( full_path ): + # Make sure the owner has execute permission on the directory. + # See http://docs.python.org/2/library/stat.html + if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: + return True return False -def assert_file_exists( full_path ): - """Return True if a file exists. This will work for both symbolic linke and files.""" - if os.path.exists( full_path ): +def assert_directory_exists( full_path ): + """ + Return True if a symbolic link or directory exists, but if full_path is a file, + return False. """ + if full_path is None: + return False + if os.path.isfile( full_path ): + return False + if os.path.isdir( full_path ): return True return False def assert_file_executable( full_path ): - """Return True if a file exists and is executable.""" - if os.path.islink( full_path ): + """ + Return True if a symbolic link or file exists and is executable, but if full_path + is a directory, return False. + """ + if full_path is None: return False - if os.path.is_file( full_path ): + if os.path.isdir( full_path ): + return False + if os.path.exists( full_path ): # Make sure the owner has execute permission on the file. # See http://docs.python.org/2/library/stat.html if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: return True return False +def assert_file_exists( full_path ): + """ + Return True if a symbolic link or file exists, but if full_path is a directory, + return False. + """ + if full_path is None: + return False + if os.path.isdir( full_path ): + return False + if os.path.exists( full_path ): + return True + return False + def create_env_var_dict( elem, tool_dependency_install_dir=None, tool_shed_repository_install_dir=None ): env_var_name = elem.get( 'name', 'PATH' ) env_var_action = elem.get( 'action', 'prepend_to' ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Refactor managers into their own module.
by commits-noreply＠bitbucket.org 27 Apr '14

27 Apr '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/56dd83576ce6/ Changeset: 56dd83576ce6 User: jmchilton Date: 2014-04-27 03:27:07 Summary: Refactor managers into their own module. Allows cleaner reuse outside of controllers. Small PEP8 fixes. Affected #: 5 files diff -r 9dbd0de27e53a2cac3c604c0aa6fb70ffb5ac7d1 -r 56dd83576ce67763863b61ad85104f8d14d978e0 lib/galaxy/managers/__init__.py --- /dev/null +++ b/lib/galaxy/managers/__init__.py @@ -0,0 +1,4 @@ +""" 'Business logic' independent of web transactions/user context (trans) +should be pushed into models - but logic that requires the context trans +should be placed under this module. +""" diff -r 9dbd0de27e53a2cac3c604c0aa6fb70ffb5ac7d1 -r 56dd83576ce67763863b61ad85104f8d14d978e0 lib/galaxy/managers/hdas.py --- /dev/null +++ b/lib/galaxy/managers/hdas.py @@ -0,0 +1,67 @@ +from galaxy import exceptions +from ..managers import histories + + +class HDAManager( object ): + + def __init__( self ): + self.histories_mgr = histories.HistoryManager() + + def get( self, trans, unencoded_id, check_ownership=True, check_accessible=True ): + """ + """ + # this is a replacement for UsesHistoryDatasetAssociationMixin because mixins are a bad soln/structure + hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( unencoded_id ) + if hda is None: + raise exceptions.ObjectNotFound() + hda = self.secure( trans, hda, check_ownership, check_accessible ) + return hda + + def secure( self, trans, hda, check_ownership=True, check_accessible=True ): + """ + checks if (a) user owns item or (b) item is accessible to user. + """ + # all items are accessible to an admin + if trans.user and trans.user_is_admin(): + return hda + if check_ownership: + hda = self.check_ownership( trans, hda ) + if check_accessible: + hda = self.check_accessible( trans, hda ) + return hda + + def can_access_dataset( self, trans, hda ): + current_user_roles = trans.get_current_user_roles() + return trans.app.security_agent.can_access_dataset( current_user_roles, hda.dataset ) + + #TODO: is_owner, is_accessible + + def check_ownership( self, trans, hda ): + if not trans.user: + #if hda.history == trans.history: + # return hda + raise exceptions.AuthenticationRequired( "Must be logged in to manage Galaxy datasets", type='error' ) + if trans.user_is_admin(): + return hda + # check for ownership of the containing history and accessibility of the underlying dataset + if( self.histories_mgr.is_owner( trans, hda.history ) + and self.can_access_dataset( trans, hda ) ): + return hda + raise exceptions.ItemOwnershipException( + "HistoryDatasetAssociation is not owned by the current user", type='error' ) + + def check_accessible( self, trans, hda ): + if trans.user and trans.user_is_admin(): + return hda + # check for access of the containing history... + self.histories_mgr.check_accessible( trans, hda.history ) + # ...then the underlying dataset + if self.can_access_dataset( trans, hda ): + return hda + raise exceptions.ItemAccessibilityException( + "HistoryDatasetAssociation is not accessible to the current user", type='error' ) + + def err_if_uploading( self, trans, hda ): + if hda.state == trans.model.Dataset.states.UPLOAD: + raise exceptions.Conflict( "Please wait until this dataset finishes uploading" ) + return hda diff -r 9dbd0de27e53a2cac3c604c0aa6fb70ffb5ac7d1 -r 56dd83576ce67763863b61ad85104f8d14d978e0 lib/galaxy/managers/histories.py --- /dev/null +++ b/lib/galaxy/managers/histories.py @@ -0,0 +1,93 @@ +from galaxy import exceptions +from galaxy.model import orm + + +class HistoryManager( object ): + #TODO: all the following would be more useful if passed the user instead of defaulting to trans.user + + def get( self, trans, unencoded_id, check_ownership=True, check_accessible=True, deleted=None ): + """ + Get a History from the database by id, verifying ownership. + """ + # this is a replacement for UsesHistoryMixin because mixins are a bad soln/structure + history = trans.sa_session.query( trans.app.model.History ).get( unencoded_id ) + if history is None: + raise exceptions.ObjectNotFound() + if deleted is True and not history.deleted: + raise exceptions.ItemDeletionException( 'History "%s" is not deleted' % ( history.name ), type="error" ) + elif deleted is False and history.deleted: + raise exceptions.ItemDeletionException( 'History "%s" is deleted' % ( history.name ), type="error" ) + + history = self.secure( trans, history, check_ownership, check_accessible ) + return history + + def by_user( self, trans, user=None, include_deleted=False, only_deleted=False ): + """ + Get all the histories for a given user (defaulting to `trans.user`) + ordered by update time and filtered on whether they've been deleted. + """ + # handle default and/or anonymous user (which still may not have a history yet) + user = user or trans.user + if not user: + current_history = trans.get_history() + return [ current_history ] if current_history else [] + + history_model = trans.model.History + query = ( trans.sa_session.query( history_model ) + .filter( history_model.user == user ) + .order_by( orm.desc( history_model.table.c.update_time ) ) ) + if only_deleted: + query = query.filter( history_model.deleted == True ) + elif not include_deleted: + query = query.filter( history_model.deleted == False ) + return query.all() + + def secure( self, trans, history, check_ownership=True, check_accessible=True ): + """ + checks if (a) user owns item or (b) item is accessible to user. + """ + # all items are accessible to an admin + if trans.user and trans.user_is_admin(): + return history + if check_ownership: + history = self.check_ownership( trans, history ) + if check_accessible: + history = self.check_accessible( trans, history ) + return history + + def is_current( self, trans, history ): + return trans.history == history + + def is_owner( self, trans, history ): + # anon users are only allowed to view their current history + if not trans.user: + return self.is_current( trans, history ) + return trans.user == history.user + + def check_ownership( self, trans, history ): + if trans.user and trans.user_is_admin(): + return history + if not trans.user and not self.is_current( trans, history ): + raise exceptions.AuthenticationRequired( "Must be logged in to manage Galaxy histories", type='error' ) + if self.is_owner( trans, history ): + return history + raise exceptions.ItemOwnershipException( "History is not owned by the current user", type='error' ) + + def is_accessible( self, trans, history ): + # admin always have access + if trans.user and trans.user_is_admin(): + return True + # owner has implicit access + if self.is_owner( trans, history ): + return True + # importable and shared histories are always accessible + if history.importable: + return True + if trans.user in history.users_shared_with_dot_users: + return True + return False + + def check_accessible( self, trans, history ): + if self.is_accessible( trans, history ): + return history + raise exceptions.ItemAccessibilityException( "History is not accessible to the current user", type='error' ) diff -r 9dbd0de27e53a2cac3c604c0aa6fb70ffb5ac7d1 -r 56dd83576ce67763863b61ad85104f8d14d978e0 lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -18,7 +18,7 @@ from galaxy.web.base.controller import ExportsHistoryMixin from galaxy.web.base.controller import ImportsHistoryMixin -from galaxy.model import orm +from galaxy.managers import histories from galaxy import util from galaxy.util import string_as_bool @@ -35,7 +35,7 @@ def __init__( self, app ): super( HistoriesController, self ).__init__( app ) self.mgrs = util.bunch.Bunch( - histories = HistoryManager() + histories=histories.HistoryManager() ) def _decode_id( self, trans, id ): @@ -404,96 +404,3 @@ pass #log.warn( 'unknown key: %s', str( key ) ) return validated_payload - - - - -class HistoryManager( object ): - #TODO: all the following would be more useful if passed the user instead of defaulting to trans.user - - def get( self, trans, unencoded_id, check_ownership=True, check_accessible=True, deleted=None ): - """ - Get a History from the database by id, verifying ownership. - """ - # this is a replacement for UsesHistoryMixin because mixins are a bad soln/structure - history = trans.sa_session.query( trans.app.model.History ).get( unencoded_id ) - if history is None: - raise exceptions.ObjectNotFound() - if deleted == True and not history.deleted: - raise exceptions.ItemDeletionException( 'History "%s" is not deleted' % ( history.name ), type="error" ) - elif deleted == False and history.deleted: - raise exceptions.ItemDeletionException( 'History "%s" is deleted' % ( history.name ), type="error" ) - - history = self.secure( trans, history, check_ownership, check_accessible ) - return history - - def by_user( self, trans, user=None, include_deleted=False, only_deleted=False ): - """ - Get all the histories for a given user (defaulting to `trans.user`) - ordered by update time and filtered on whether they've been deleted. - """ - # handle default and/or anonymous user (which still may not have a history yet) - user = user or trans.user - if not user: - current_history = trans.get_history() - return [ current_history ] if current_history else [] - - history_model = trans.model.History - query = ( trans.sa_session.query( history_model ) - .filter( history_model.user == user ) - .order_by( orm.desc( history_model.table.c.update_time ) ) ) - if only_deleted: - query = query.filter( history_model.deleted == True ) - elif not include_deleted: - query = query.filter( history_model.deleted == False ) - return query.all() - - def secure( self, trans, history, check_ownership=True, check_accessible=True ): - """ - checks if (a) user owns item or (b) item is accessible to user. - """ - # all items are accessible to an admin - if trans.user and trans.user_is_admin(): - return history - if check_ownership: - history = self.check_ownership( trans, history ) - if check_accessible: - history = self.check_accessible( trans, history ) - return history - - def is_current( self, trans, history ): - return trans.history == history - - def is_owner( self, trans, history ): - # anon users are only allowed to view their current history - if not trans.user: - return self.is_current( trans, history ) - return trans.user == history.user - - def check_ownership( self, trans, history ): - if trans.user and trans.user_is_admin(): - return history - if not trans.user and not self.is_current( trans, history ): - raise exceptions.AuthenticationRequired( "Must be logged in to manage Galaxy histories", type='error' ) - if self.is_owner( trans, history ): - return history - raise exceptions.ItemOwnershipException( "History is not owned by the current user", type='error' ) - - def is_accessible( self, trans, history ): - # admin always have access - if trans.user and trans.user_is_admin(): - return True - # owner has implicit access - if self.is_owner( trans, history ): - return True - # importable and shared histories are always accessible - if history.importable: - return True - if trans.user in history.users_shared_with_dot_users: - return True - return False - - def check_accessible( self, trans, history ): - if self.is_accessible( trans, history ): - return history - raise exceptions.ItemAccessibilityException( "History is not accessible to the current user", type='error' ) diff -r 9dbd0de27e53a2cac3c604c0aa6fb70ffb5ac7d1 -r 56dd83576ce67763863b61ad85104f8d14d978e0 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -7,7 +7,6 @@ from galaxy.web import _future_expose_api as expose_api from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous -from galaxy.web import _future_expose_api_raw as expose_api_raw from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin @@ -18,7 +17,8 @@ from galaxy.web.base.controller import url_for -from galaxy.webapps.galaxy.api import histories +from galaxy.managers import histories +from galaxy.managers import hdas import logging log = logging.getLogger( __name__ ) @@ -30,8 +30,8 @@ def __init__( self, app ): super( HistoryContentsController, self ).__init__( app ) self.mgrs = util.bunch.Bunch( - histories = histories.HistoryManager(), - hdas = HDAManager() + histories=histories.HistoryManager(), + hdas=hdas.HDAManager() ) def _decode_id( self, trans, id ): @@ -434,67 +434,3 @@ def __handle_unknown_contents_type( self, trans, contents_type ): raise exceptions.UnknownContentsType('Unknown contents type: %s' % type) - -class HDAManager( object ): - - def __init__( self ): - self.histories_mgr = histories.HistoryManager() - - def get( self, trans, unencoded_id, check_ownership=True, check_accessible=True ): - """ - """ - # this is a replacement for UsesHistoryDatasetAssociationMixin because mixins are a bad soln/structure - hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( unencoded_id ) - if hda is None: - raise exceptions.ObjectNotFound() - hda = self.secure( trans, hda, check_ownership, check_accessible ) - return hda - - def secure( self, trans, hda, check_ownership=True, check_accessible=True ): - """ - checks if (a) user owns item or (b) item is accessible to user. - """ - # all items are accessible to an admin - if trans.user and trans.user_is_admin(): - return hda - if check_ownership: - hda = self.check_ownership( trans, hda ) - if check_accessible: - hda = self.check_accessible( trans, hda ) - return hda - - def can_access_dataset( self, trans, hda ): - current_user_roles = trans.get_current_user_roles() - return trans.app.security_agent.can_access_dataset( current_user_roles, hda.dataset ) - - #TODO: is_owner, is_accessible - - def check_ownership( self, trans, hda ): - if not trans.user: - #if hda.history == trans.history: - # return hda - raise exceptions.AuthenticationRequired( "Must be logged in to manage Galaxy datasets", type='error' ) - if trans.user_is_admin(): - return hda - # check for ownership of the containing history and accessibility of the underlying dataset - if( self.histories_mgr.is_owner( trans, hda.history ) - and self.can_access_dataset( trans, hda ) ): - return hda - raise exceptions.ItemOwnershipException( - "HistoryDatasetAssociation is not owned by the current user", type='error' ) - - def check_accessible( self, trans, hda ): - if trans.user and trans.user_is_admin(): - return hda - # check for access of the containing history... - self.histories_mgr.check_accessible( trans, hda.history ) - # ...then the underlying dataset - if self.can_access_dataset( trans, hda ): - return hda - raise exceptions.ItemAccessibilityException( - "HistoryDatasetAssociation is not accessible to the current user", type='error' ) - - def err_if_uploading( self, trans, hda ): - if hda.state == trans.model.Dataset.states.UPLOAD: - raise exceptions.Conflict( "Please wait until this dataset finishes uploading" ) - return hda Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 27 Apr '14

27 Apr '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4d4ad2a65454/ Changeset: 4d4ad2a65454 User: jmchilton Date: 2014-04-27 00:52:08 Summary: Tools API Tests - Add another test with repeat. Adding abstractions reused heavily in collections and multi-execution tests downstream. Affected #: 1 file diff -r 03efdcca196906f3aac9939517096da445ff0bf3 -r 4d4ad2a654545e461c3b41c903a53657a640c07d test/functional/api/test_tools.py --- a/test/functional/api/test_tools.py +++ b/test/functional/api/test_tools.py @@ -54,28 +54,69 @@ self.assertEquals( result_content, table ) def test_run_cat1( self ): + # Run simple non-upload tool with an input data parameter. history_id = self._new_history() - new_dataset = self._new_dataset( history_id ) - dataset_id = new_dataset[ 'id' ] + new_dataset = self._new_dataset( history_id, content='Cat1Test' ) + inputs = dict( + input1=dataset_to_param( new_dataset ), + ) + outputs = self._cat1_outputs( history_id, inputs=inputs ) + self.assertEquals( len( outputs ), 1 ) + self._wait_for_history( history_id, assert_ok=True ) + output1 = outputs[ 0 ] + output1_content = self._get_content( history_id, dataset=output1 ) + self.assertEqual( output1_content.strip(), "Cat1Test" ) + + def test_run_cat1_with_two_inputs( self ): + # Run tool with an multiple data parameter and grouping (repeat) + history_id = self._new_history() + new_dataset1 = self._new_dataset( history_id, content='Cat1Test' ) + new_dataset2 = self._new_dataset( history_id, content='Cat2Test' ) + inputs = { + 'input1': dataset_to_param( new_dataset1 ), + 'queries_0|input2': dataset_to_param( new_dataset2 ) + } + outputs = self._cat1_outputs( history_id, inputs=inputs ) + self.assertEquals( len( outputs ), 1 ) + self._wait_for_history( history_id, assert_ok=True ) + output1 = outputs[ 0 ] + output1_content = self._get_content( history_id, dataset=output1 ) + self.assertEqual( output1_content.strip(), "Cat1Test\nCat2Test" ) + + def _cat1_outputs( self, history_id, inputs ): + create_response = self._run_cat1( history_id, inputs ) + self._assert_status_code_is( create_response, 200 ) + create = create_response.json() + self._assert_has_keys( create, 'outputs' ) + return create[ 'outputs' ] + + def _run_cat1( self, history_id, inputs ): payload = self._run_tool_payload( tool_id='cat1', - inputs=dict( - input1=dict( - src='hda', - id=dataset_id - ), - ), + inputs=inputs, history_id=history_id, ) create_response = self._post( "tools", data=payload ) - self._assert_status_code_is( create_response, 200 ) - self._assert_has_keys( create_response.json(), 'outputs' ) - self._wait_for_history( history_id, assert_ok=True ) + return create_response def _upload_and_get_content( self, content, **upload_kwds ): history_id = self._new_history() new_dataset = self._new_dataset( history_id, content=content, **upload_kwds ) self._wait_for_history( history_id, assert_ok=True ) - display_response = self._get( "histories/%s/contents/%s/display" % ( history_id, new_dataset[ "id" ] ) ) + return self._get_content( history_id, dataset=new_dataset ) + + def _get_content( self, history_id, **kwds ): + if "dataset_id" in kwds: + dataset_id = kwds[ "dataset_id" ] + else: + dataset_id = kwds[ "dataset" ][ "id" ] + display_response = self._get( "histories/%s/contents/%s/display" % ( history_id, dataset_id ) ) self._assert_status_code_is( display_response, 200 ) return display_response.content + + +def dataset_to_param( dataset ): + return dict( + src='hda', + id=dataset[ 'id' ] + ) https://bitbucket.org/galaxy/galaxy-central/commits/a05d14c84adb/ Changeset: a05d14c84adb User: jmchilton Date: 2014-04-27 00:52:08 Summary: Tools API Tests - Refactor away from deprecated mixin toward DatasetPopulator class. Affected #: 2 files diff -r 4d4ad2a654545e461c3b41c903a53657a640c07d -r a05d14c84adb736daaeb80323269964a0fab97ff test/functional/api/helpers.py --- a/test/functional/api/helpers.py +++ b/test/functional/api/helpers.py @@ -9,6 +9,8 @@ # row - first grabbing 8 lines at random and then 6. workflow_random_x2_str = resource_string( __name__, "test_workflow_2.ga" ) +DEFAULT_HISTORY_TIMEOUT = 5 # Secs to wait on history to turn ok + # Deprecated mixin, use dataset populator instead. # TODO: Rework existing tests to target DatasetPopulator in a setup method instead. @@ -40,8 +42,8 @@ run_response = self.galaxy_interactor.post( "tools", data=payload ) return run_response.json()["outputs"][0] - def wait_for_history( self, history_id, assert_ok=False ): - wait_on_state( lambda: self.galaxy_interactor.get( "histories/%s" % history_id ), assert_ok=assert_ok ) + def wait_for_history( self, history_id, assert_ok=False, timeout=DEFAULT_HISTORY_TIMEOUT ): + wait_on_state( lambda: self.galaxy_interactor.get( "histories/%s" % history_id ), assert_ok=assert_ok, timeout=timeout ) def new_history( self, **kwds ): name = kwds.get( "name", "API Test History" ) diff -r 4d4ad2a654545e461c3b41c903a53657a640c07d -r a05d14c84adb736daaeb80323269964a0fab97ff test/functional/api/test_tools.py --- a/test/functional/api/test_tools.py +++ b/test/functional/api/test_tools.py @@ -3,10 +3,14 @@ from base import api from operator import itemgetter -from .helpers import TestsDatasets +from .helpers import DatasetPopulator -class ToolsTestCase( api.ApiTestCase, TestsDatasets ): +class ToolsTestCase( api.ApiTestCase ): + + def setUp( self ): + super( ToolsTestCase, self ).setUp( ) + self.dataset_populator = DatasetPopulator( self.galaxy_interactor ) def test_index( self ): index = self._get( "tools" ) @@ -27,8 +31,8 @@ assert "cat1" in tool_ids def test_upload1_paste( self ): - history_id = self._new_history() - payload = self._upload_payload( history_id, 'Hello World' ) + history_id = self.dataset_populator.new_history() + payload = self.dataset_populator.upload_payload( history_id, 'Hello World' ) create_response = self._post( "tools", data=payload ) self._assert_has_keys( create_response.json(), 'outputs' ) @@ -55,30 +59,30 @@ def test_run_cat1( self ): # Run simple non-upload tool with an input data parameter. - history_id = self._new_history() - new_dataset = self._new_dataset( history_id, content='Cat1Test' ) + history_id = self.dataset_populator.new_history() + new_dataset = self.dataset_populator.new_dataset( history_id, content='Cat1Test' ) inputs = dict( input1=dataset_to_param( new_dataset ), ) outputs = self._cat1_outputs( history_id, inputs=inputs ) self.assertEquals( len( outputs ), 1 ) - self._wait_for_history( history_id, assert_ok=True ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) output1 = outputs[ 0 ] output1_content = self._get_content( history_id, dataset=output1 ) self.assertEqual( output1_content.strip(), "Cat1Test" ) def test_run_cat1_with_two_inputs( self ): # Run tool with an multiple data parameter and grouping (repeat) - history_id = self._new_history() - new_dataset1 = self._new_dataset( history_id, content='Cat1Test' ) - new_dataset2 = self._new_dataset( history_id, content='Cat2Test' ) + history_id = self.dataset_populator.new_history() + new_dataset1 = self.dataset_populator.new_dataset( history_id, content='Cat1Test' ) + new_dataset2 = self.dataset_populator.new_dataset( history_id, content='Cat2Test' ) inputs = { 'input1': dataset_to_param( new_dataset1 ), 'queries_0|input2': dataset_to_param( new_dataset2 ) } outputs = self._cat1_outputs( history_id, inputs=inputs ) self.assertEquals( len( outputs ), 1 ) - self._wait_for_history( history_id, assert_ok=True ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) output1 = outputs[ 0 ] output1_content = self._get_content( history_id, dataset=output1 ) self.assertEqual( output1_content.strip(), "Cat1Test\nCat2Test" ) @@ -91,7 +95,7 @@ return create[ 'outputs' ] def _run_cat1( self, history_id, inputs ): - payload = self._run_tool_payload( + payload = self.dataset_populator.run_tool_payload( tool_id='cat1', inputs=inputs, history_id=history_id, @@ -100,9 +104,9 @@ return create_response def _upload_and_get_content( self, content, **upload_kwds ): - history_id = self._new_history() - new_dataset = self._new_dataset( history_id, content=content, **upload_kwds ) - self._wait_for_history( history_id, assert_ok=True ) + history_id = self.dataset_populator.new_history() + new_dataset = self.dataset_populator.new_dataset( history_id, content=content, **upload_kwds ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) return self._get_content( history_id, dataset=new_dataset ) def _get_content( self, history_id, **kwds ): https://bitbucket.org/galaxy/galaxy-central/commits/9dbd0de27e53/ Changeset: 9dbd0de27e53 User: jmchilton Date: 2014-04-27 00:52:08 Summary: Workflow Editor Unit Tests - Improvements to input terminal canAccept tests. Use real connectors, add tests for logic related to multiple input data parameters. Affected #: 1 file diff -r a05d14c84adb736daaeb80323269964a0fab97ff -r 9dbd0de27e53a2cac3c604c0aa6fb70ffb5ac7d1 test/qunit/tests/workflow_editor_tests.js --- a/test/qunit/tests/workflow_editor_tests.js +++ b/test/qunit/tests/workflow_editor_tests.js @@ -60,20 +60,27 @@ }; module( "Input terminal model test", { - setup: function() { - this.node = { }; - this.element = $( "<div>" ); - var input = { extensions: [ "txt" ], multiple: false }; - this.input_terminal = new InputTerminal( { element: this.element, input: input } ); + setup: function( ) { + this.node = new Node( { } ); + this.input = { extensions: [ "txt" ], multiple: false }; + this.input_terminal = new InputTerminal( { input: this.input } ); this.input_terminal.node = this.node; }, - test_connector: function( attr ) { - var connector = attr || {}; - this.input_terminal.connectors.push( connector ); + multiple: function( ) { + this.input.multiple = true; + this.input_terminal.update( this.input ); + }, + test_connector: function( ) { + var outputTerminal = new OutputTerminal( { datatypes: [ 'input' ] } ); + var inputTerminal = this.input_terminal; + var connector; + with_workflow_global( function() { + connector = new Connector( outputTerminal, inputTerminal ); + } ); return connector; }, - with_test_connector: function( attr, f ) { - this.test_connector( attr ); + with_test_connector: function( f ) { + this.test_connector( ); f(); this.reset_connectors(); }, @@ -116,7 +123,7 @@ test( "test disconnect", function() { this.node.markChanged = sinon.spy(); - var connector = this.test_connector( {} ); + var connector = this.test_connector( ); this.input_terminal.disconnect( connector ); // Assert node markChanged called @@ -126,17 +133,19 @@ } ); test( "test redraw", function() { - var connector = this.test_connector( { redraw: sinon.spy() } ); + var connector = this.test_connector( ); + connector.redraw = sinon.spy(); this.input_terminal.redraw(); // Assert connectors were redrawn ok( connector.redraw.called ); } ); test( "test destroy", function() { - var connector = this.test_connector( { destroy: sinon.spy() } ); + var connector = this.test_connector(); + connector.destroy = sinon.spy(); this.input_terminal.destroy(); - // Assert connectors were redrawn + // Assert connectors were destroyed ok( connector.destroy.called ); } ); @@ -189,11 +198,29 @@ test( "cannot accept when already connected", function() { var self = this; // If other is subtype but already connected, cannot accept - this.with_test_connector( {}, function() { + this.with_test_connector( function() { ok( ! self.test_accept() ); } ); } ); + test( "can accept already connected inputs if input is multiple", function() { + var self = this; + this.multiple(); + this.with_test_connector( function() { + ok( self.test_accept() ); + } ); + } ); + + test( "cannot accept already connected inputs if input is multiple but datatypes don't match", function() { + var other = { node: {}, datatypes: [ "binary" ] }; // binary is not txt + + var self = this; + this.multiple(); + this.with_test_connector( function() { + ok( ! self.test_accept( other ) ); + } ); + } ); + module( "Connector test", { } ); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: martenson: data libraries API: improved deleted folder items handling, pydocs
by commits-noreply＠bitbucket.org 25 Apr '14

25 Apr '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/03efdcca1969/ Changeset: 03efdcca1969 User: martenson Date: 2014-04-25 23:44:50 Summary: data libraries API: improved deleted folder items handling, pydocs Affected #: 1 file diff -r de882c9035116eccd7563c9f89aec586a017ce2f -r 03efdcca196906f3aac9939517096da445ff0bf3 lib/galaxy/webapps/galaxy/api/folder_contents.py --- a/lib/galaxy/webapps/galaxy/api/folder_contents.py +++ b/lib/galaxy/webapps/galaxy/api/folder_contents.py @@ -2,6 +2,7 @@ API operations on the contents of a library folder. """ from galaxy import web +from galaxy import util from galaxy import exceptions from galaxy.web import _future_expose_api as expose_api from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous @@ -21,10 +22,32 @@ def index( self, trans, folder_id, **kwd ): """ GET /api/folders/{encoded_folder_id}/contents - Displays a collection (list) of a folder's contents (files and folders). - Encoded folder ID is prepended with 'F' if it is a folder as opposed to a data set which does not have it. - Full path is provided in response as a separate object providing data for breadcrumb path building. + + Displays a collection (list) of a folder's contents + (files and folders). Encoded folder ID is prepended + with 'F' if it is a folder as opposed to a data set + which does not have it. Full path is provided in + response as a separate object providing data for + breadcrumb path building. + + :param folder_id: encoded ID of the folder which + contents should be library_dataset_dict + :type folder_id: encoded string + + :param kwd: keyword dictionary with other params + :type kwd: dict + + :returns: dictionary containing all items and metadata + :type: dict + + :raises: MalformedId, InconsistentDatabase, ObjectNotFound, + InternalServerError """ + deleted = kwd.get( 'deleted', 'missing' ) + try: + deleted = util.asbool( deleted ) + except ValueError: + deleted = False if ( len( folder_id ) == 17 and folder_id.startswith( 'F' ) ): try: @@ -69,7 +92,14 @@ def build_path( folder ): """ - Search the path upwards recursively and load the whole route of names and ids for breadcrumb building purposes. + Search the path upwards recursively and load the whole route of + names and ids for breadcrumb building purposes. + + :param folder: current folder for navigating up + :param type: Galaxy LibraryFolder + + :returns: list consisting of full path to the library + :type: list """ path_to_root = [] # We are almost in root @@ -89,7 +119,7 @@ update_time = '' create_time = '' # Go through every accessible item in the folder and include its meta-data. - for content_item in self._load_folder_contents( trans, folder ): + for content_item in self._load_folder_contents( trans, folder, deleted ): can_access = trans.app.security_agent.can_access_library_item( current_user_roles, content_item, trans.user ) if ( can_access or ( content_item.api_type == 'folder' and trans.app.security_agent.folder_is_unrestricted( content_item ) ) ): return_item = {} @@ -119,31 +149,74 @@ type = content_item.api_type, name = content_item.name, update_time = update_time, - create_time = create_time + create_time = create_time, + deleted = content_item.deleted ) ) folder_contents.append( return_item ) return { 'metadata' : { 'full_path' : full_path, 'can_add_library_item': can_add_library_item, 'folder_name': folder.name }, 'folder_contents' : folder_contents } - def _load_folder_contents( self, trans, folder ): + def _load_folder_contents( self, trans, folder, include_deleted ): """ - Loads all contents of the folder (folders and data sets) but only in the first level. + Loads all contents of the folder (folders and data sets) but only + in the first level. Include deleted if the flag is set and if the + user has access to undelete it. + + :param folder: the folder which contents are being loaded + :type folder: Galaxy LibraryFolder + + :param include_deleted: flag, when true the items that are deleted + and can be undeleted by current user are shown + :type include_deleted: boolean + + :returns: a list containing the requested items + :type: list """ current_user_roles = trans.get_current_user_roles() is_admin = trans.user_is_admin() content_items = [] for subfolder in folder.active_folders: - if not is_admin: - can_access, folder_ids = trans.app.security_agent.check_folder_contents( trans.user, current_user_roles, subfolder ) - if (is_admin or can_access) and not subfolder.deleted: - subfolder.api_type = 'folder' - content_items.append( subfolder ) + if subfolder.deleted: + if include_deleted: + if is_admin: + subfolder.api_type = 'folder' + content_items.append( subfolder ) + else: + can_modify = trans.app.security_agent.can_modify_library_item( current_user_roles, subfolder ) + if can_modify: + subfolder.api_type = 'folder' + content_items.append( subfolder ) + else: + if is_admin: + subfolder.api_type = 'folder' + content_items.append( subfolder ) + else: + can_access, folder_ids = trans.app.security_agent.check_folder_contents( trans.user, current_user_roles, subfolder ) + if can_access: + subfolder.api_type = 'folder' + content_items.append( subfolder ) + for dataset in folder.datasets: - if not is_admin: - can_access = trans.app.security_agent.can_access_dataset( current_user_roles, dataset.library_dataset_dataset_association.dataset ) - if (is_admin or can_access) and not dataset.deleted: - dataset.api_type = 'file' - content_items.append( dataset ) + if dataset.deleted: + if include_deleted: + if is_admin: + dataset.api_type = 'file' + content_items.append( dataset ) + else: + can_modify = trans.app.security_agent.can_modify_library_item( current_user_roles, dataset ) + if can_modify: + dataset.api_type = 'file' + content_items.append( dataset ) + else: + if is_admin: + dataset.api_type = 'file' + content_items.append( dataset ) + else: + can_access, folder_ids = trans.app.security_agent.can_access_dataset( current_user_roles, dataset.library_dataset_dataset_association.dataset ) + if can_access: + dataset.api_type = 'file' + content_items.append( dataset ) + return content_items @expose_api @@ -157,15 +230,22 @@ :type payload: dict * folder_id: the parent folder of the new item - * from_hda_id: (optional) the id of an accessible HDA to copy into the library - * ldda_message: (optional) the new message attribute of the LDDA created - * extended_metadata: (optional) dub-dictionary containing any extended - metadata to associate with the item + * from_hda_id: (optional) the id of an accessible HDA to copy + into the library + * ldda_message: (optional) the new message attribute of the LDDA + created + * extended_metadata: (optional) dub-dictionary containing any + extended metadata to associate with the item - :returns: a dictionary containing the id, name, and 'show' url of the new item + :returns: a dictionary containing the id, name, + and 'show' url of the new item :rtype: dict + + :raises: ObjectAttributeInvalidException, + InsufficientPermissionsException, ItemAccessibilityException, + InternalServerError """ - class_name, encoded_folder_id_16 = self.__decode_library_content_id( trans, encoded_folder_id ) + encoded_folder_id_16 = self.__decode_library_content_id( trans, encoded_folder_id ) from_hda_id, ldda_message = ( payload.pop( 'from_hda_id', None ), payload.pop( 'ldda_message', '' ) ) if ldda_message: ldda_message = util.sanitize_html.sanitize_html( ldda_message, 'utf-8' ) @@ -200,8 +280,21 @@ return rval def __decode_library_content_id( self, trans, encoded_folder_id ): + """ + Identifies whether the id provided is properly encoded + LibraryFolder. + + :param encoded_folder_id: encoded id of Galaxy LibraryFolder + :type encoded_folder_id: encoded string + + :returns: last 16 chars of the encoded id in case it was Folder + (had 'F' prepended) + :type: string + + :raises: MalformedId + """ if ( len( encoded_folder_id ) == 17 and encoded_folder_id.startswith( 'F' )): - return 'LibraryFolder', encoded_folder_id[1:] + return encoded_folder_id[1:] else: raise exceptions.MalformedId( 'Malformed folder id ( %s ) specified, unable to decode.' % str( encoded_folder_id ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0