March 2012 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: james_taylor: style: various improvements for menubutton styles
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/528ca90fbd3e/ changeset: 528ca90fbd3e user: james_taylor date: 2012-03-29 22:27:22 summary: style: various improvements for menubutton styles affected #: 2 files diff -r 7e82d5412edc1617da7ebdabd0d511a96348bcbc -r 528ca90fbd3e629bbe1eba92dc0faffb600a556f static/june_2007_style/base.less --- a/static/june_2007_style/base.less +++ b/static/june_2007_style/base.less @@ -1222,6 +1222,10 @@ .menubutton { .btn(); + &:hover { .btn.hover(); } + &:active { .btn.active(); } + &:focus { .tab-focus(); } + display: inline-block; cursor: pointer; position: relative; @@ -1235,11 +1239,21 @@ // padding: 1px 0.25em; // margin: -1px -0.25em; + a { + text-decoration: none; + } + .label { position: relative; - // display: block; + display: inline-block; border-right: none; text-decoration: none; + text-align: left; + // The following properties truncate the text and force the button to have one line + max-height: 2*@baseLineHeight; + line-height: @baseLineHeight; + overflow: hidden; + text-overflow: ellipsis; } &.popup .label { @@ -1247,21 +1261,16 @@ padding-right: 6px; } - &.popup { + &.popup, &.popup.split { + padding-right: 18px; &:after { margin-top: 6px; + position: absolute; + top: 2px; + right: 6px; .caret(); } } - - &.popup.split { - &:after { - margin-top: 6px; - margin-left: 0px; - .caret(); - } - } - } // A split menu button, the main button has an action, the arrow causes the diff -r 7e82d5412edc1617da7ebdabd0d511a96348bcbc -r 528ca90fbd3e629bbe1eba92dc0faffb600a556f static/june_2007_style/blue/base.css --- a/static/june_2007_style/blue/base.css +++ b/static/june_2007_style/blue/base.css @@ -661,10 +661,13 @@ .action-button:active{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);-moz-box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);background-color:#e6e6e6;background-color:#d9d9d9 \9;outline:0;} .menubutton{display:inline-block;padding:2px 10px 2px;font-size:12px;line-height:16px;color:#111111;text-align:center;text-shadow:0 1px 1px rgba(255, 255, 255, 0.75);background-color:#fafafa;background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), color-stop(25%, #ffffff), to(#e6e6e6));background-image:-webkit-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);background-image:-moz-linear-gradient(top, #ffffff, #ffffff 25%, #e6e6e6);background-image:-ms-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);background-image:-o-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);background-image:linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);background-repeat:no-repeat;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffff', endColorstr='#e6e6e6', GradientType=0);border:1px solid #999999;border-bottom-color:#888888;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.2),0 1px 2px rgba(0, 0, 0, 0.05);-moz-box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.2),0 1px 2px rgba(0, 0, 0, 0.05);box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.2),0 1px 2px rgba(0, 0, 0, 0.05);cursor:pointer;*margin-left:.3em;display:inline-block;cursor:pointer;position:relative;-webkit-user-select:none;-moz-user-select:none;-o-user-select:none;user-select:none;}.menubutton:first-child{*margin-left:0;} .menubutton [class^="ficon"],.menubutton [class*=" ficon"]{line-height:.9em;} -.menubutton .label{position:relative;border-right:none;text-decoration:none;} +.menubutton:hover{color:#111111;text-decoration:none;background-color:#e6e6e6;background-position:0 -15px;-webkit-transition:background-position 0.1s linear;-moz-transition:background-position 0.1s linear;-ms-transition:background-position 0.1s linear;-o-transition:background-position 0.1s linear;transition:background-position 0.1s linear;} +.menubutton:active{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);-moz-box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);background-color:#e6e6e6;background-color:#d9d9d9 \9;outline:0;} +.menubutton:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px;} +.menubutton a{text-decoration:none;} +.menubutton .label{position:relative;display:inline-block;border-right:none;text-decoration:none;text-align:left;max-height:32px;line-height:16px;overflow:hidden;text-overflow:ellipsis;} .menubutton.popup .label{border-right:solid #999999 1px;padding-right:6px;} -.menubutton.popup:after{margin-top:6px;display:inline-block;width:0;height:0;text-indent:-99999px;*text-indent:0;vertical-align:top;border-left:4px solid transparent;border-right:4px solid transparent;border-top:4px solid #000000;opacity:0.3;filter:alpha(opacity=30);content:"\2193";} -.menubutton.popup.split:after{margin-top:6px;margin-left:0px;display:inline-block;width:0;height:0;text-indent:-99999px;*text-indent:0;vertical-align:top;border-left:4px solid transparent;border-right:4px solid transparent;border-top:4px solid #000000;opacity:0.3;filter:alpha(opacity=30);content:"\2193";} +.menubutton.popup,.menubutton.popup.split{padding-right:18px;}.menubutton.popup:after,.menubutton.popup.split:after{margin-top:6px;position:absolute;top:2px;right:6px;display:inline-block;width:0;height:0;text-indent:-99999px;*text-indent:0;vertical-align:top;border-left:4px solid transparent;border-right:4px solid transparent;border-top:4px solid #000000;opacity:0.3;filter:alpha(opacity=30);content:"\2193";} div.popmenu-wrapper{position:absolute;top:100%;z-index:20000;}div.popmenu-wrapper ul.dropdown-menu{display:block;position:relative;float:none;} ul.dropdown-menu a{text-decoration:none;} ul.dropdown-menu li.head>a{text-shadow:0 1px 0 rgba(255, 255, 255, 0.5);font-size:11px;font-weight:bold;line-height:16px;color:#999999;text-transform:uppercase;} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: james_taylor: style: fix brand overflowing
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/7e82d5412edc/ changeset: 7e82d5412edc user: james_taylor date: 2012-03-29 20:52:58 summary: style: fix brand overflowing affected #: 3 files diff -r 59f0cdeb9afcd899727ba2d2b5b4f3d0e6383edd -r 7e82d5412edc1617da7ebdabd0d511a96348bcbc static/june_2007_style/base.less --- a/static/june_2007_style/base.less +++ b/static/june_2007_style/base.less @@ -407,14 +407,26 @@ } .title { - .navbar.brand(); + + position: absolute; + left: 0; + top: 0; + font-family: verdana; font-weight: bold; + font-size: 20px; + line-height: 1; color: white; // Override margin and padding due to shorter navbar height padding: 5px 20px 12px; margin-left: -15px; z-index: 2000; + + img { + display: inline; + width: 26px; + vertical-align: top; + } a { color: white; text-decoration: none; diff -r 59f0cdeb9afcd899727ba2d2b5b4f3d0e6383edd -r 7e82d5412edc1617da7ebdabd0d511a96348bcbc static/june_2007_style/blue/base.css --- a/static/june_2007_style/blue/base.css +++ b/static/june_2007_style/blue/base.css @@ -225,7 +225,7 @@ .navbar-form .radio,.navbar-form .checkbox{margin-top:5px;} .navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px;} .navbar-search{position:relative;float:left;margin-top:6px;margin-bottom:0;}.navbar-search .search-query{padding:4px 9px;font-family:"Lucida Grande",verdana,arial,helvetica,sans-serif;font-size:13px;font-weight:normal;line-height:1;color:#ffffff;color:rgba(255, 255, 255, 0.75);background:#666;background:rgba(255, 255, 255, 0.3);border:1px solid #111;-webkit-box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1),0 1px 0px rgba(255, 255, 255, 0.15);-moz-box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1),0 1px 0px rgba(255, 255, 255, 0.15);box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1),0 1px 0px rgba(255, 255, 255, 0.15);-webkit-transition:none;-moz-transition:none;-ms-transition:none;-o-transition:none;transition:none;}.navbar-search .search-query :-moz-placeholder{color:#eeeeee;} -.navbar-search .search-query ::-webkit-input-placeholder{color:#eeeeee;} +.navbar-search .search-query::-webkit-input-placeholder{color:#eeeeee;} .navbar-search .search-query:hover{color:#ffffff;background-color:#999999;background-color:rgba(255, 255, 255, 0.5);} .navbar-search .search-query:focus,.navbar-search .search-query.focused{padding:5px 10px;color:#333333;text-shadow:0 1px 0 #ffffff;background-color:#ffffff;border:0;-webkit-box-shadow:0 0 3px rgba(0, 0, 0, 0.15);-moz-box-shadow:0 0 3px rgba(0, 0, 0, 0.15);box-shadow:0 0 3px rgba(0, 0, 0, 0.15);outline:0;} .navbar-fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030;} @@ -535,7 +535,8 @@ #masthead .btn-group .btn{margin-top:0;} #masthead .nav>li>a{padding:6px 10px 8px;cursor:pointer;}#masthead .nav>li>a:hover{color:gold;} #masthead .dropdown-menu a,#masthead .dropdown-menu a:hover{text-decoration:none;} -#masthead .title{float:left;display:block;padding:8px 20px 12px;margin-left:-20px;font-size:20px;font-weight:200;line-height:1;color:#ffffff;font-family:verdana;font-weight:bold;color:white;padding:5px 20px 12px;margin-left:-15px;z-index:2000;}#masthead .title a{color:white;text-decoration:none;} +#masthead .title{position:absolute;left:0;top:0;font-family:verdana;font-weight:bold;font-size:20px;line-height:1;color:white;padding:5px 20px 12px;margin-left:-15px;z-index:2000;}#masthead .title img{display:inline;width:26px;vertical-align:top;} +#masthead .title a{color:white;text-decoration:none;} #masthead .masthead-inner{padding-left:20px;padding-right:20px;background-color:#303239;background-image:-moz-linear-gradient(top, #333333, #2c3143);background-image:-ms-linear-gradient(top, #333333, #2c3143);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#333333), to(#2c3143));background-image:-webkit-linear-gradient(top, #333333, #2c3143);background-image:-o-linear-gradient(top, #333333, #2c3143);background-image:linear-gradient(top, #333333, #2c3143);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#333333', endColorstr='#2c3143', GradientType=0);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 3px rgba(0, 0, 0, 0.25),inset 0 -1px 0 rgba(0, 0, 0, 0.1);-moz-box-shadow:0 1px 3px rgba(0, 0, 0, 0.25),inset 0 -1px 0 rgba(0, 0, 0, 0.1);box-shadow:0 1px 3px rgba(0, 0, 0, 0.25),inset 0 -1px 0 rgba(0, 0, 0, 0.1);padding-left:0;padding-right:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;height:32px;} #masthead a:hover{text-decoration:underline;} .quota-meter-container{position:absolute;top:0;right:0;height:32px;} diff -r 59f0cdeb9afcd899727ba2d2b5b4f3d0e6383edd -r 7e82d5412edc1617da7ebdabd0d511a96348bcbc templates/webapps/galaxy/base_panels.mako --- a/templates/webapps/galaxy/base_panels.mako +++ b/templates/webapps/galaxy/base_panels.mako @@ -184,12 +184,12 @@ </div> ## Logo, layered over tabs to be clickable - <div class="title" style="position: absolute; top: 0; left: 0; white-space: nowrap;"> + <div class="title"><a href="${app.config.get( 'logo_url', '/' )}"> - <img border="0" src="${h.url_for('/static/images/galaxyIcon_noText.png')}" style="display: inline; width: 26px; vertical-align: top;"> + <img border="0" src="${h.url_for('/static/images/galaxyIcon_noText.png')}"> Galaxy %if app.config.brand: - <span class='brand'>/ ${app.config.brand}</span> + <span>/ ${app.config.brand}</span> %endif </a></div> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the functional test framework to support testing tools contained in installed repositories that were not migrated from the Galaxy distribution. To test these tools, use: sh run_functional_tests.sh --installed
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/59f0cdeb9afc/ changeset: 59f0cdeb9afc user: greg date: 2012-03-29 21:02:36 summary: Enhance the functional test framework to support testing tools contained in installed repositories that were not migrated from the Galaxy distribution. To test these tools, use: sh run_functional_tests.sh --installed affected #: 4 files diff -r c510097f7018dbc177513a62c0ca46b4cace0c86 -r 59f0cdeb9afcd899727ba2d2b5b4f3d0e6383edd run_functional_tests.sh --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -29,6 +29,15 @@ else python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html --migrated fi +elif [ $1 = '--installed' ]; then + if [ ! $2 ]; then + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html --installed + elif [ $2 = '-id' ]; then + # TODO: This option is not tested... + python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$3 --with-nosehtml --html-report-file run_functional_tests.html --installed + else + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html --installed + fi else python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html $1 fi diff -r c510097f7018dbc177513a62c0ca46b4cace0c86 -r 59f0cdeb9afcd899727ba2d2b5b4f3d0e6383edd scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -48,21 +48,65 @@ default_galaxy_locales = 'en' default_galaxy_test_file_dir = "test-data" migrated_tool_panel_config = 'migrated_tools_conf.xml' +installed_tool_panel_configs = [ 'shed_tool_conf.xml' ] + +def parse_tool_panel_config( config, shed_tools_dict ): + """ + Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed. + """ + last_galaxy_test_file_dir = None + last_tested_repository_name = None + last_tested_changeset_revision = None + tree = util.parse_xml( config ) + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + galaxy_test_file_dir, \ + last_tested_repository_name, \ + last_tested_changeset_revision = get_installed_repository_info( elem, + last_galaxy_test_file_dir, + last_tested_repository_name, + last_tested_changeset_revision ) + if galaxy_test_file_dir: + if galaxy_test_file_dir != last_galaxy_test_file_dir: + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + guid = elem.get( 'guid' ) + shed_tools_dict[ guid ] = galaxy_test_file_dir + last_galaxy_test_file_dir = galaxy_test_file_dir + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + galaxy_test_file_dir, \ + last_tested_repository_name, \ + last_tested_changeset_revision = get_installed_repository_info( section_elem, + last_galaxy_test_file_dir, + last_tested_repository_name, + last_tested_changeset_revision ) + if galaxy_test_file_dir: + if galaxy_test_file_dir != last_galaxy_test_file_dir: + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + guid = section_elem.get( 'guid' ) + shed_tools_dict[ guid ] = galaxy_test_file_dir + last_galaxy_test_file_dir = galaxy_test_file_dir + return shed_tools_dict def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision ): """ Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem. - This only happens when testing tools eliminated from the distribution and now installed from the tool shed. + This only happens when testing tools installed from the tool shed. """ tool_config_path = elem.get( 'file' ) installed_tool_path_items = tool_config_path.split( '/repos/' ) sans_shed = installed_tool_path_items[ 1 ] path_items = sans_shed.split( '/' ) + repository_owner = path_items[ 0 ] repository_name = path_items[ 1 ] changeset_revision = path_items[ 2 ] if repository_name != last_tested_repository_name or changeset_revision != last_tested_changeset_revision: # Locate the test-data directory. - installed_tool_path = os.path.join( installed_tool_path_items[ 0 ], 'repos', 'devteam', repository_name, changeset_revision ) + installed_tool_path = os.path.join( installed_tool_path_items[ 0 ], 'repos', repository_owner, repository_name, changeset_revision ) for root, dirs, files in os.walk( installed_tool_path ): if 'test-data' in dirs: return os.path.join( root, 'test-data' ), repository_name, changeset_revision @@ -92,12 +136,12 @@ if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales testing_migrated_tools = '--migrated' in sys.argv + testing_installed_tools = '--installed' in sys.argv - if testing_migrated_tools: + if testing_migrated_tools or testing_installed_tools: sys.argv.pop() # Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs. - galaxy_migrated_tools_file = 'migrated_tools_dict' - migrated_tools_dict = {} + galaxy_tool_shed_test_file = 'shed_tools_dict' # We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it. fd, tmp_tool_panel_conf = tempfile.mkstemp() os.write( fd, '<?xml version="1.0"?>\n' ) @@ -127,7 +171,7 @@ tool_data_table_config_path = 'tool_data_table_conf.xml' tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None ) use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False ) - + if start_server: psu_production = False galaxy_test_proxy_port = None @@ -295,54 +339,32 @@ os.environ[ 'GALAXY_TEST_SAVE' ] = galaxy_test_save # Pass in through script setenv, will leave a copy of ALL test validate files os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host - if testing_migrated_tools: - last_galaxy_test_file_dir = None - last_tested_repository_name = None - last_tested_changeset_revision = None - tree = util.parse_xml( migrated_tool_panel_config ) - root = tree.getroot() - migrated_tool_path = root.get( 'tool_path' ) - counter = 0 - for elem in root: - if elem.tag == 'tool': - galaxy_test_file_dir, \ - last_tested_repository_name, \ - last_tested_changeset_revision = get_installed_repository_info( elem, - last_galaxy_test_file_dir, - last_tested_repository_name, - last_tested_changeset_revision ) - if galaxy_test_file_dir: - if galaxy_test_file_dir != last_galaxy_test_file_dir: - if not os.path.isabs( galaxy_test_file_dir ): - galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) - guid = elem.get( 'guid' ) - migrated_tools_dict[ guid ] = galaxy_test_file_dir - last_galaxy_test_file_dir = galaxy_test_file_dir - elif elem.tag == 'section': - for section_elem in elem: - if section_elem.tag == 'tool': - galaxy_test_file_dir, \ - last_tested_repository_name, \ - last_tested_changeset_revision = get_installed_repository_info( section_elem, - last_galaxy_test_file_dir, - last_tested_repository_name, - last_tested_changeset_revision ) - if galaxy_test_file_dir: - if galaxy_test_file_dir != last_galaxy_test_file_dir: - if not os.path.isabs( galaxy_test_file_dir ): - galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) - guid = section_elem.get( 'guid' ) - migrated_tools_dict[ guid ] = galaxy_test_file_dir - last_galaxy_test_file_dir = galaxy_test_file_dir - # Persist the migrated_tools_dict to the galaxy_migrated_tools_file. - migrated_tools_file = open( galaxy_migrated_tools_file, 'w' ) - migrated_tools_file.write( to_json_string( migrated_tools_dict ) ) - migrated_tools_file.close() - if not os.path.isabs( galaxy_migrated_tools_file ): - galaxy_migrated_tools_file = os.path.join( os.getcwd(), galaxy_migrated_tools_file ) - os.environ[ 'GALAXY_MIGRATED_TOOLS_FILE' ] = galaxy_migrated_tools_file + if testing_migrated_tools or testing_installed_tools: + shed_tools_dict = {} + if testing_migrated_tools: + shed_tools_dict = parse_tool_panel_config( migrated_tool_panel_config, shed_tools_dict ) + elif testing_installed_tools: + for shed_tool_config in installed_tool_panel_configs: + shed_tools_dict = parse_tool_panel_config( shed_tool_config, shed_tools_dict ) + # Persist the shed_tools_dict to the galaxy_tool_shed_test_file. + shed_tools_file = open( galaxy_tool_shed_test_file, 'w' ) + shed_tools_file.write( to_json_string( shed_tools_dict ) ) + shed_tools_file.close() + if not os.path.isabs( galaxy_tool_shed_test_file ): + galaxy_tool_shed_test_file = os.path.join( os.getcwd(), galaxy_tool_shed_test_file ) + os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_tool_shed_test_file + if testing_installed_tools: + # Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs, + # and reload the app's toolbox. + relative_migrated_tool_panel_config = os.path.join( app.config.root, migrated_tool_panel_config ) + tool_configs = app.config.tool_configs + if relative_migrated_tool_panel_config in tool_configs: + tool_configs.remove( relative_migrated_tool_panel_config ) + for installed_tool_panel_config in installed_tool_panel_configs: + tool_configs.append( installed_tool_panel_config ) + app.toolbox = tools.ToolBox( tool_configs, app.config.tool_path, app ) functional.test_toolbox.toolbox = app.toolbox - functional.test_toolbox.build_tests( testing_migrated_tools=True ) + functional.test_toolbox.build_tests( testing_shed_tools=True ) test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) result = run_tests( test_config ) @@ -352,9 +374,9 @@ except: log.info( "Unable to remove temporary file: %s" % tmp_tool_panel_conf ) try: - os.unlink( galaxy_migrated_tools_file ) + os.unlink( galaxy_tool_shed_test_file ) except: - log.info( "Unable to remove file: %s" % galaxy_migrated_tools_file ) + log.info( "Unable to remove file: %s" % galaxy_tool_shed_test_file ) else: functional.test_toolbox.toolbox = app.toolbox functional.test_toolbox.build_tests() diff -r c510097f7018dbc177513a62c0ca46b4cace0c86 -r 59f0cdeb9afcd899727ba2d2b5b4f3d0e6383edd test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -36,14 +36,14 @@ self.port = os.environ.get( 'GALAXY_TEST_PORT' ) self.url = "http://%s:%s" % ( self.host, self.port ) self.file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', None ) - self.migrated_tools_file = os.environ.get( 'GALAXY_MIGRATED_TOOLS_FILE', None ) - if self.migrated_tools_file: - f = open( self.migrated_tools_file, 'r' ) + self.tool_shed_test_file = os.environ.get( 'GALAXY_TOOL_SHED_TEST_FILE', None ) + if self.tool_shed_test_file: + f = open( self.tool_shed_test_file, 'r' ) text = f.read() f.close() - self.migrated_tools_dict = from_json_string( text ) + self.shed_tools_dict = from_json_string( text ) else: - self.migrated_tools_dict = {} + self.shed_tools_dict = {} self.keepOutdir = os.environ.get( 'GALAXY_TEST_SAVE', '' ) if self.keepOutdir > '': try: @@ -169,9 +169,9 @@ if line_diff_count > lines_diff: raise AssertionError, "Failed to find '%s' in history data. (lines_diff=%i):\n" % ( contains, lines_diff ) - def get_filename( self, filename, migrated_tool_id=None ): - if migrated_tool_id and self.migrated_tools_dict: - file_dir = self.migrated_tools_dict[ migrated_tool_id ] + def get_filename( self, filename, shed_tool_id=None ): + if shed_tool_id and self.shed_tools_dict: + file_dir = self.shed_tools_dict[ shed_tool_id ] if not file_dir: file_dir = self.file_dir else: @@ -183,9 +183,9 @@ filename = os.path.join( *path ) file(filename, 'wt').write(buffer.getvalue()) - def upload_file( self, filename, ftype='auto', dbkey='unspecified (?)', space_to_tab=False, metadata=None, composite_data=None, migrated_tool_id=None ): + def upload_file( self, filename, ftype='auto', dbkey='unspecified (?)', space_to_tab=False, metadata=None, composite_data=None, shed_tool_id=None ): """ - Uploads a file. If migrated_tool_id has a value, we're testing tools migrated from the distribution to the tool shed, + Uploads a file. If shed_tool_id has a value, we're testing tools migrated from the distribution to the tool shed, so the tool-data directory of test data files is contained in the installed tool shed repository. """ self.visit_url( "%s/tool_runner?tool_id=upload1" % self.url ) @@ -197,11 +197,11 @@ tc.fv( "1", "files_metadata|%s" % elem.get( 'name' ), elem.get( 'value' ) ) if composite_data: for i, composite_file in enumerate( composite_data ): - filename = self.get_filename( composite_file.get( 'value' ), migrated_tool_id=migrated_tool_id ) + filename = self.get_filename( composite_file.get( 'value' ), shed_tool_id=shed_tool_id ) tc.formfile( "1", "files_%i|file_data" % i, filename ) tc.fv( "1", "files_%i|space_to_tab" % i, composite_file.get( 'space_to_tab', False ) ) else: - filename = self.get_filename( filename, migrated_tool_id=migrated_tool_id ) + filename = self.get_filename( filename, shed_tool_id=shed_tool_id ) tc.formfile( "1", "file_data", filename ) tc.fv( "1", "space_to_tab", space_to_tab ) tc.submit("runtool_btn") @@ -653,7 +653,7 @@ fd,temp_prefix = tempfile.mkstemp(prefix='tmp',suffix=suffix) return temp_prefix - def verify_dataset_correctness( self, filename, hid=None, wait=True, maxseconds=120, attributes=None, migrated_tool_id=None ): + def verify_dataset_correctness( self, filename, hid=None, wait=True, maxseconds=120, attributes=None, shed_tool_id=None ): """Verifies that the attributes and contents of a history item meet expectations""" if wait: self.wait( maxseconds=maxseconds ) #wait for job to finish @@ -692,7 +692,7 @@ errmsg += str( err ) raise AssertionError( errmsg ) if filename is not None: - local_name = self.get_filename( filename, migrated_tool_id=migrated_tool_id ) + local_name = self.get_filename( filename, shed_tool_id=shed_tool_id ) temp_name = self.makeTfname(fname = filename) file( temp_name, 'wb' ).write(data) if self.keepOutdir > '': @@ -726,7 +726,7 @@ else: raise Exception, 'Unimplemented Compare type: %s' % compare if extra_files: - self.verify_extra_files_content( extra_files, elem.get( 'id' ), migrated_tool_id=migrated_tool_id ) + self.verify_extra_files_content( extra_files, elem.get( 'id' ), shed_tool_id=shed_tool_id ) except AssertionError, err: errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare ) errmsg += str( err ) @@ -745,21 +745,21 @@ os.remove( temp_name ) return temp_local, temp_temp - def verify_extra_files_content( self, extra_files, hda_id, migrated_tool_id=None ): + def verify_extra_files_content( self, extra_files, hda_id, shed_tool_id=None ): files_list = [] for extra_type, extra_value, extra_name, extra_attributes in extra_files: if extra_type == 'file': files_list.append( ( extra_name, extra_value, extra_attributes ) ) elif extra_type == 'directory': - for filename in os.listdir( self.get_filename( extra_value, migrated_tool_id=migrated_tool_id ) ): + for filename in os.listdir( self.get_filename( extra_value, shed_tool_id=shed_tool_id ) ): files_list.append( ( filename, os.path.join( extra_value, filename ), extra_attributes ) ) else: raise ValueError, 'unknown extra_files type: %s' % extra_type for filename, filepath, attributes in files_list: - self.verify_composite_datatype_file_content( filepath, hda_id, base_name=filename, attributes=attributes, migrated_tool_id=migrated_tool_id ) + self.verify_composite_datatype_file_content( filepath, hda_id, base_name=filename, attributes=attributes, shed_tool_id=shed_tool_id ) - def verify_composite_datatype_file_content( self, file_name, hda_id, base_name=None, attributes=None, migrated_tool_id=None ): - local_name = self.get_filename( file_name, migrated_tool_id=migrated_tool_id ) + def verify_composite_datatype_file_content( self, file_name, hda_id, base_name=None, attributes=None, shed_tool_id=None ): + local_name = self.get_filename( file_name, shed_tool_id=shed_tool_id ) if base_name is None: base_name = os.path.split(file_name)[-1] temp_name = self.makeTfname(fname = base_name) @@ -1015,8 +1015,8 @@ def last_page( self ): return tc.browser.get_html() - def load_cookies( self, file, migrated_tool_id=None ): - filename = self.get_filename( file, migrated_tool_id=migrated_tool_id ) + def load_cookies( self, file, shed_tool_id=None ): + filename = self.get_filename( file, shed_tool_id=shed_tool_id ) tc.load_cookies(filename) def reload_page( self ): diff -r c510097f7018dbc177513a62c0ca46b4cace0c86 -r 59f0cdeb9afcd899727ba2d2b5b4f3d0e6383edd test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -10,7 +10,7 @@ class ToolTestCase( TwillTestCase ): """Abstract test case that runs tests based on a `galaxy.tools.test.ToolTest`""" - def do_it( self, testdef, migrated_tool_id=None ): + def do_it( self, testdef, shed_tool_id=None ): # If the test generation had an error, raise if testdef.error: if testdef.exception: @@ -40,7 +40,7 @@ dbkey=extra.get( 'dbkey', 'hg17' ), metadata=metadata, composite_data=composite_data, - migrated_tool_id=migrated_tool_id ) + shed_tool_id=shed_tool_id ) print "Uploaded file: ", fname, ", ftype: ", extra.get( 'ftype', 'auto' ), ", extra: ", extra #Post upload attribute editing edit_attributes = extra.get( 'edit_attributes', [] ) @@ -99,7 +99,7 @@ elem_hid = elem.get( 'hid' ) elem_index += 1 try: - self.verify_dataset_correctness( outfile, hid=elem_hid, maxseconds=testdef.maxseconds, attributes=attributes, migrated_tool_id=migrated_tool_id ) + self.verify_dataset_correctness( outfile, hid=elem_hid, maxseconds=testdef.maxseconds, attributes=attributes, shed_tool_id=shed_tool_id ) except Exception, e: print >>sys.stderr, self.get_job_stdout( elem.get( 'id' ), format=True ) print >>sys.stderr, self.get_job_stderr( elem.get( 'id' ), format=True ) @@ -143,7 +143,7 @@ expanded_inputs[value.name] = declared_inputs[value.name] return expanded_inputs -def build_tests( testing_migrated_tools=False ): +def build_tests( testing_shed_tools=False ): """ If the module level variable `toolbox` is set, generate `ToolTestCase` classes for all of its tests and put them into this modules globals() so @@ -166,12 +166,12 @@ baseclasses = ( ToolTestCase, ) namespace = dict() for j, testdef in enumerate( tool.tests ): - def make_test_method( td, migrated_tool_id=None ): + def make_test_method( td, shed_tool_id=None ): def test_tool( self ): - self.do_it( td, migrated_tool_id=migrated_tool_id ) + self.do_it( td, shed_tool_id=shed_tool_id ) return test_tool - if testing_migrated_tools: - test_method = make_test_method( testdef, migrated_tool_id=tool.id ) + if testing_shed_tools: + test_method = make_test_method( testdef, shed_tool_id=tool.id ) else: test_method = make_test_method( testdef ) test_method.__doc__ = "%s ( %s ) > %s" % ( tool.name, tool.id, testdef.name ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the functional test framework to support tools migrated from the distribution and installed from the tool shed. To test these tools, use: sh run_functional_tests.sh --migrated
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/c510097f7018/ changeset: c510097f7018 user: greg date: 2012-03-29 18:03:14 summary: Enhance the functional test framework to support tools migrated from the distribution and installed from the tool shed. To test these tools, use: sh run_functional_tests.sh --migrated affected #: 6 files diff -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a -r c510097f7018dbc177513a62c0ca46b4cace0c86 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -276,7 +276,6 @@ root = tree.getroot() tool_path = root.get( 'tool_path', None ) if tool_path is None: - # There will be a problem here if the user has defined 2 non-shed related configs. config_filenames.append( config_filename ) return config_filenames def __get_url_from_tool_shed( self, tool_shed ): diff -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a -r c510097f7018dbc177513a62c0ca46b4cace0c86 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -124,7 +124,7 @@ elif elem.tag == 'section': self.load_section_tag_set( elem, tool_path, load_panel_dict ) elif elem.tag == 'label': - self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel ) + self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict ) if parsing_shed_tool_conf: shed_tool_conf_dict = dict( config_filename=config_filename, tool_path=tool_path, @@ -370,10 +370,10 @@ integrated_panel_dict[ key ] = workflow except: log.exception( "Error loading workflow: %s" % workflow_id ) - def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict ): + def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict ): label = ToolSectionLabel( elem ) key = 'label_' + label.id - if not self.integrated_tool_panel_config_has_contents: + if load_panel_dict: panel_dict[ key ] = label integrated_panel_dict[ key ] = label def load_section_tag_set( self, elem, tool_path, load_panel_dict ): @@ -396,7 +396,7 @@ elif sub_elem.tag == 'workflow': self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict ) elif sub_elem.tag == 'label': - self.load_label_tag_set( sub_elem, elems, integrated_elems ) + self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict ) if load_panel_dict: self.tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. diff -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a -r c510097f7018dbc177513a62c0ca46b4cace0c86 run_functional_tests.sh --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -20,6 +20,15 @@ echo "===========================================================================================================================================" echo "'run_functional_tests.sh -id bbb' for testing one tool with id 'bbb' ('bbb' is the tool id)" echo "'run_functional_tests.sh -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" +elif [ $1 = '--migrated' ]; then + if [ ! $2 ]; then + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html --migrated + elif [ $2 = '-id' ]; then + # TODO: This option is not tested... + python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$3 --with-nosehtml --html-report-file run_functional_tests.html --migrated + else + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html --migrated + fi else python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html $1 fi diff -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a -r c510097f7018dbc177513a62c0ca46b4cace0c86 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import os, sys, shutil +import os, sys, shutil, tempfile, re # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() @@ -32,6 +32,13 @@ from galaxy.web import buildapp from galaxy import tools from galaxy.util import bunch +from galaxy import util +from galaxy.util.json import to_json_string + +import nose.core +import nose.config +import nose.loader +import nose.plugins.manager log = logging.getLogger( "functional_tests.py" ) @@ -40,28 +47,87 @@ default_galaxy_test_port_max = 9999 default_galaxy_locales = 'en' default_galaxy_test_file_dir = "test-data" +migrated_tool_panel_config = 'migrated_tools_conf.xml' -def main(): - +def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision ): + """ + Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem. + This only happens when testing tools eliminated from the distribution and now installed from the tool shed. + """ + tool_config_path = elem.get( 'file' ) + installed_tool_path_items = tool_config_path.split( '/repos/' ) + sans_shed = installed_tool_path_items[ 1 ] + path_items = sans_shed.split( '/' ) + repository_name = path_items[ 1 ] + changeset_revision = path_items[ 2 ] + if repository_name != last_tested_repository_name or changeset_revision != last_tested_changeset_revision: + # Locate the test-data directory. + installed_tool_path = os.path.join( installed_tool_path_items[ 0 ], 'repos', 'devteam', repository_name, changeset_revision ) + for root, dirs, files in os.walk( installed_tool_path ): + if 'test-data' in dirs: + return os.path.join( root, 'test-data' ), repository_name, changeset_revision + return None, repository_name, changeset_revision + return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision + +def run_tests( test_config ): + loader = nose.loader.TestLoader( config=test_config ) + plug_loader = test_config.plugins.prepareTestLoader( loader ) + if plug_loader is not None: + loader = plug_loader + tests = loader.loadTestsFromNames( test_config.testNames ) + test_runner = nose.core.TextTestRunner( stream=test_config.stream, + verbosity=test_config.verbosity, + config=test_config ) + plug_runner = test_config.plugins.prepareTestRunner( test_runner ) + if plug_runner is not None: + test_runner = plug_runner + return test_runner.run( tests ) + +def main(): # ---- Configuration ------------------------------------------------------ - galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host ) galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', None ) galaxy_test_save = os.environ.get( 'GALAXY_TEST_SAVE', None) + tool_path = os.environ.get( 'GALAXY_TEST_TOOL_PATH', 'tools' ) + if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: + os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales + testing_migrated_tools = '--migrated' in sys.argv - if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: - os.environ['HTTP_ACCEPT_LANGUAGE'] = default_galaxy_locales - galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir ) - if not os.path.isabs( galaxy_test_file_dir ): - galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) - start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ - tool_path = os.environ.get( 'GALAXY_TEST_TOOL_PATH', 'tools' ) - tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml.sample' ) - tool_data_table_config_path = 'tool_data_table_conf.xml' + if testing_migrated_tools: + sys.argv.pop() + # Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs. + galaxy_migrated_tools_file = 'migrated_tools_dict' + migrated_tools_dict = {} + # We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it. + fd, tmp_tool_panel_conf = tempfile.mkstemp() + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<toolbox>\n' ) + os.write( fd, '<tool file="data_source/upload.xml"/>\n' ) + os.write( fd, '</toolbox>\n' ) + os.close( fd ) + tool_config_file = tmp_tool_panel_conf + galaxy_test_file_dir = None + library_import_dir = None + user_library_import_dir = None + # Exclude all files except test_toolbox.py. + ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) ) + else: + tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml.sample' ) + galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir ) + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + library_import_dir = galaxy_test_file_dir + user_library_import_dir = os.path.join( galaxy_test_file_dir, 'users' ) + ignore_files = () + + start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ + if os.path.exists( 'tool_data_table_conf.test.xml' ): + tool_data_table_config_path = 'tool_data_table_conf.test.xml' + else: + tool_data_table_config_path = 'tool_data_table_conf.xml' tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None ) use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False ) - if os.path.exists( 'tool_data_table_conf.test.xml' ): - tool_data_table_config_path = 'tool_data_table_conf.test.xml' + if start_server: psu_production = False galaxy_test_proxy_port = None @@ -126,38 +192,21 @@ try: os.makedirs( dir ) except OSError: - pass - + pass print "Database connection:", database_connection - - # What requires these? - # handy for (eg) functional tests to save outputs? - if galaxy_test_save: - os.environ['GALAXY_TEST_SAVE'] = galaxy_test_save - # pass in through script setenv - # will leave a copy of ALL test validate files - os.environ['GALAXY_TEST_HOST'] = galaxy_test_host - os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir - # ---- Build Application -------------------------------------------------- - - app = None - + # ---- Build Application -------------------------------------------------- + app = None if start_server: - global_conf = { '__file__' : 'universe_wsgi.ini.sample' } if psu_production: global_conf = None - if not database_connection.startswith( 'sqlite://' ): - kwargs['database_engine_option_max_overflow'] = '20' - + kwargs[ 'database_engine_option_max_overflow' ] = '20' if tool_dependency_dir is not None: - kwargs['tool_dependency_dir'] = tool_dependency_dir - + kwargs[ 'tool_dependency_dir' ] = tool_dependency_dir if use_distributed_object_store: - kwargs['object_store'] = 'distributed' - kwargs['distributed_object_store_config_file'] = 'distributed_object_store_conf.xml.sample' - + kwargs[ 'object_store' ] = 'distributed' + kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample' # Build the Universe Application app = UniverseApplication( job_queue_workers = 5, id_secret = 'changethisinproductiontoo', @@ -179,21 +228,16 @@ allow_user_deletion = True, admin_users = 'test(a)bx.psu.edu', allow_library_path_paste = True, - library_import_dir = galaxy_test_file_dir, - user_library_import_dir = os.path.join( galaxy_test_file_dir, 'users' ), + library_import_dir = library_import_dir, + user_library_import_dir = user_library_import_dir, global_conf = global_conf, **kwargs ) - - log.info( "Embedded Universe application started" ); - + log.info( "Embedded Universe application started" ) # ---- Run webserver ------------------------------------------------------ - server = None if start_server: - - webapp = buildapp.app_factory( dict(), use_translogger = False, static_enabled = False, app=app ) - + webapp = buildapp.app_factory( dict(), use_translogger=False, static_enabled=False, app=app ) if galaxy_test_port is not None: server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) else: @@ -217,7 +261,6 @@ t = threading.Thread( target=server.serve_forever ) t.start() - # Test if the server is up for i in range( 10 ): conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port ) # directly test the app, not the proxy @@ -227,85 +270,105 @@ time.sleep( 0.1 ) else: raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" ) - # Test if the proxy server is up if psu_production: conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_proxy_port ) # directly test the app, not the proxy conn.request( "GET", "/" ) if not conn.getresponse().status == 200: raise Exception( "Test HTTP proxy server did not return '200 OK'" ) - log.info( "Embedded web server started" ) - - # ---- Load toolbox for generated tests ----------------------------------- - # We don't add the tests to the path until everything is up and running new_path = [ os.path.join( cwd, "test" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path - import functional.test_toolbox - - if app: - # TODO: provisions for loading toolbox from file when using external server - functional.test_toolbox.toolbox = app.toolbox - functional.test_toolbox.build_tests() - else: - # FIXME: This doesn't work at all now that toolbox requires an 'app' instance - # (to get at datatypes, might just pass a datatype registry directly) - datatypes_registry = galaxy.datatypes.registry.Registry() - datatypes_registry.load_datatypes() - my_app = bunch.Bunch( datatypes_registry ) - test_toolbox.toolbox = tools.ToolBox( 'tool_conf.xml.test', 'tools', my_app ) - # ---- Find tests --------------------------------------------------------- - if galaxy_test_proxy_port: log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_proxy_port ) ) else: log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) - success = False - try: - - import nose.core - import nose.config - import nose.loader - import nose.plugins.manager - - test_config = nose.config.Config( env = os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - - loader = nose.loader.TestLoader( config = test_config ) - - plug_loader = test_config.plugins.prepareTestLoader( loader ) - if plug_loader is not None: - loader = plug_loader - - tests = loader.loadTestsFromNames( test_config.testNames ) - - test_runner = nose.core.TextTestRunner( - stream = test_config.stream, - verbosity = test_config.verbosity, - config = test_config) - - plug_runner = test_config.plugins.prepareTestRunner( test_runner ) - if plug_runner is not None: - test_runner = plug_runner - - result = test_runner.run( tests ) - - success = result.wasSuccessful() - + # What requires these? Handy for (eg) functional tests to save outputs? + if galaxy_test_save: + os.environ[ 'GALAXY_TEST_SAVE' ] = galaxy_test_save + # Pass in through script setenv, will leave a copy of ALL test validate files + os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host + if testing_migrated_tools: + last_galaxy_test_file_dir = None + last_tested_repository_name = None + last_tested_changeset_revision = None + tree = util.parse_xml( migrated_tool_panel_config ) + root = tree.getroot() + migrated_tool_path = root.get( 'tool_path' ) + counter = 0 + for elem in root: + if elem.tag == 'tool': + galaxy_test_file_dir, \ + last_tested_repository_name, \ + last_tested_changeset_revision = get_installed_repository_info( elem, + last_galaxy_test_file_dir, + last_tested_repository_name, + last_tested_changeset_revision ) + if galaxy_test_file_dir: + if galaxy_test_file_dir != last_galaxy_test_file_dir: + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + guid = elem.get( 'guid' ) + migrated_tools_dict[ guid ] = galaxy_test_file_dir + last_galaxy_test_file_dir = galaxy_test_file_dir + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + galaxy_test_file_dir, \ + last_tested_repository_name, \ + last_tested_changeset_revision = get_installed_repository_info( section_elem, + last_galaxy_test_file_dir, + last_tested_repository_name, + last_tested_changeset_revision ) + if galaxy_test_file_dir: + if galaxy_test_file_dir != last_galaxy_test_file_dir: + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + guid = section_elem.get( 'guid' ) + migrated_tools_dict[ guid ] = galaxy_test_file_dir + last_galaxy_test_file_dir = galaxy_test_file_dir + # Persist the migrated_tools_dict to the galaxy_migrated_tools_file. + migrated_tools_file = open( galaxy_migrated_tools_file, 'w' ) + migrated_tools_file.write( to_json_string( migrated_tools_dict ) ) + migrated_tools_file.close() + if not os.path.isabs( galaxy_migrated_tools_file ): + galaxy_migrated_tools_file = os.path.join( os.getcwd(), galaxy_migrated_tools_file ) + os.environ[ 'GALAXY_MIGRATED_TOOLS_FILE' ] = galaxy_migrated_tools_file + functional.test_toolbox.toolbox = app.toolbox + functional.test_toolbox.build_tests( testing_migrated_tools=True ) + test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + result = run_tests( test_config ) + success = result.wasSuccessful() + try: + os.unlink( tmp_tool_panel_conf ) + except: + log.info( "Unable to remove temporary file: %s" % tmp_tool_panel_conf ) + try: + os.unlink( galaxy_migrated_tools_file ) + except: + log.info( "Unable to remove file: %s" % galaxy_migrated_tools_file ) + else: + functional.test_toolbox.toolbox = app.toolbox + functional.test_toolbox.build_tests() + if galaxy_test_file_dir: + os.environ[ 'GALAXY_TEST_FILE_DIR' ] = galaxy_test_file_dir + test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + result = run_tests( test_config ) + success = result.wasSuccessful() except: log.exception( "Failure running tests" ) log.info( "Shutting down" ) - - # ---- Teardown ----------------------------------------------------------- - + # ---- Tear down ----------------------------------------------------------- if server: log.info( "Shutting down embedded web server" ) server.server_close() @@ -330,7 +393,6 @@ shutil.rmtree( dir ) except: pass - if success: return 0 else: diff -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a -r c510097f7018dbc177513a62c0ca46b4cace0c86 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -39,8 +39,9 @@ self.migrated_tools_file = os.environ.get( 'GALAXY_MIGRATED_TOOLS_FILE', None ) if self.migrated_tools_file: f = open( self.migrated_tools_file, 'r' ) - self.migrated_tools_dict = from_json_string( f.readlines() ) + text = f.read() f.close() + self.migrated_tools_dict = from_json_string( text ) else: self.migrated_tools_dict = {} self.keepOutdir = os.environ.get( 'GALAXY_TEST_SAVE', '' ) @@ -51,8 +52,6 @@ pass self.home() - #self.set_history() - # Functions associated with files def files_diff( self, file1, file2, attributes=None ): """Checks the contents of 2 files for differences""" @@ -170,17 +169,25 @@ if line_diff_count > lines_diff: raise AssertionError, "Failed to find '%s' in history data. (lines_diff=%i):\n" % ( contains, lines_diff ) - def get_filename( self, filename ): - full = os.path.join( self.file_dir, filename) - return os.path.abspath(full) + def get_filename( self, filename, migrated_tool_id=None ): + if migrated_tool_id and self.migrated_tools_dict: + file_dir = self.migrated_tools_dict[ migrated_tool_id ] + if not file_dir: + file_dir = self.file_dir + else: + file_dir = self.file_dir + return os.path.abspath( os.path.join( file_dir, filename ) ) def save_log( *path ): """Saves the log to a file""" filename = os.path.join( *path ) file(filename, 'wt').write(buffer.getvalue()) - def upload_file( self, filename, ftype='auto', dbkey='unspecified (?)', space_to_tab = False, metadata = None, composite_data = None ): - """Uploads a file""" + def upload_file( self, filename, ftype='auto', dbkey='unspecified (?)', space_to_tab=False, metadata=None, composite_data=None, migrated_tool_id=None ): + """ + Uploads a file. If migrated_tool_id has a value, we're testing tools migrated from the distribution to the tool shed, + so the tool-data directory of test data files is contained in the installed tool shed repository. + """ self.visit_url( "%s/tool_runner?tool_id=upload1" % self.url ) try: self.refresh_form( "file_type", ftype ) #Refresh, to support composite files @@ -190,11 +197,11 @@ tc.fv( "1", "files_metadata|%s" % elem.get( 'name' ), elem.get( 'value' ) ) if composite_data: for i, composite_file in enumerate( composite_data ): - filename = self.get_filename( composite_file.get( 'value' ) ) + filename = self.get_filename( composite_file.get( 'value' ), migrated_tool_id=migrated_tool_id ) tc.formfile( "1", "files_%i|file_data" % i, filename ) tc.fv( "1", "files_%i|space_to_tab" % i, composite_file.get( 'space_to_tab', False ) ) else: - filename = self.get_filename( filename ) + filename = self.get_filename( filename, migrated_tool_id=migrated_tool_id ) tc.formfile( "1", "file_data", filename ) tc.fv( "1", "space_to_tab", space_to_tab ) tc.submit("runtool_btn") @@ -212,6 +219,7 @@ raise AssertionError, "Invalid hid (%s) created when uploading file %s" % ( hid, filename ) # Wait for upload processing to finish (TODO: this should be done in each test case instead) self.wait() + def upload_url_paste( self, url_paste, ftype='auto', dbkey='unspecified (?)' ): """Pasted data in the upload utility""" self.visit_page( "tool_runner/index?tool_id=upload1" ) @@ -620,6 +628,7 @@ check_str = '1 dataset copied to 1 history' self.check_page_for_string( check_str ) self.home() + def get_hids_in_history( self ): """Returns the list of hid values for items in a history""" data_list = self.get_history_as_data_list() @@ -628,6 +637,7 @@ hid = elem.get('hid') hids.append(hid) return hids + def get_hids_in_histories( self ): """Returns the list of hids values for items in all histories""" data_list = self.get_histories_as_data_list() @@ -643,7 +653,7 @@ fd,temp_prefix = tempfile.mkstemp(prefix='tmp',suffix=suffix) return temp_prefix - def verify_dataset_correctness( self, filename, hid=None, wait=True, maxseconds=120, attributes=None ): + def verify_dataset_correctness( self, filename, hid=None, wait=True, maxseconds=120, attributes=None, migrated_tool_id=None ): """Verifies that the attributes and contents of a history item meet expectations""" if wait: self.wait( maxseconds=maxseconds ) #wait for job to finish @@ -682,7 +692,7 @@ errmsg += str( err ) raise AssertionError( errmsg ) if filename is not None: - local_name = self.get_filename( filename ) + local_name = self.get_filename( filename, migrated_tool_id=migrated_tool_id ) temp_name = self.makeTfname(fname = filename) file( temp_name, 'wb' ).write(data) if self.keepOutdir > '': @@ -716,7 +726,7 @@ else: raise Exception, 'Unimplemented Compare type: %s' % compare if extra_files: - self.verify_extra_files_content( extra_files, elem.get( 'id' ) ) + self.verify_extra_files_content( extra_files, elem.get( 'id' ), migrated_tool_id=migrated_tool_id ) except AssertionError, err: errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare ) errmsg += str( err ) @@ -735,21 +745,21 @@ os.remove( temp_name ) return temp_local, temp_temp - def verify_extra_files_content( self, extra_files, hda_id ): + def verify_extra_files_content( self, extra_files, hda_id, migrated_tool_id=None ): files_list = [] for extra_type, extra_value, extra_name, extra_attributes in extra_files: if extra_type == 'file': files_list.append( ( extra_name, extra_value, extra_attributes ) ) elif extra_type == 'directory': - for filename in os.listdir( self.get_filename( extra_value ) ): + for filename in os.listdir( self.get_filename( extra_value, migrated_tool_id=migrated_tool_id ) ): files_list.append( ( filename, os.path.join( extra_value, filename ), extra_attributes ) ) else: raise ValueError, 'unknown extra_files type: %s' % extra_type for filename, filepath, attributes in files_list: - self.verify_composite_datatype_file_content( filepath, hda_id, base_name = filename, attributes = attributes ) + self.verify_composite_datatype_file_content( filepath, hda_id, base_name=filename, attributes=attributes, migrated_tool_id=migrated_tool_id ) - def verify_composite_datatype_file_content( self, file_name, hda_id, base_name = None, attributes = None ): - local_name = self.get_filename( file_name ) + def verify_composite_datatype_file_content( self, file_name, hda_id, base_name=None, attributes=None, migrated_tool_id=None ): + local_name = self.get_filename( file_name, migrated_tool_id=migrated_tool_id ) if base_name is None: base_name = os.path.split(file_name)[-1] temp_name = self.makeTfname(fname = base_name) @@ -1005,8 +1015,8 @@ def last_page( self ): return tc.browser.get_html() - def load_cookies( self, file ): - filename = self.get_filename(file) + def load_cookies( self, file, migrated_tool_id=None ): + filename = self.get_filename( file, migrated_tool_id=migrated_tool_id ) tc.load_cookies(filename) def reload_page( self ): diff -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a -r c510097f7018dbc177513a62c0ca46b4cace0c86 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -10,7 +10,7 @@ class ToolTestCase( TwillTestCase ): """Abstract test case that runs tests based on a `galaxy.tools.test.ToolTest`""" - def do_it( self, testdef ): + def do_it( self, testdef, migrated_tool_id=None ): # If the test generation had an error, raise if testdef.error: if testdef.exception: @@ -35,7 +35,12 @@ children = extra.get( 'children', [] ) metadata = extra.get( 'metadata', [] ) composite_data = extra.get( 'composite_data', [] ) - self.upload_file( fname, ftype=extra.get( 'ftype', 'auto' ), dbkey=extra.get( 'dbkey', 'hg17' ), metadata = metadata, composite_data = composite_data ) + self.upload_file( fname, + ftype=extra.get( 'ftype', 'auto' ), + dbkey=extra.get( 'dbkey', 'hg17' ), + metadata=metadata, + composite_data=composite_data, + migrated_tool_id=migrated_tool_id ) print "Uploaded file: ", fname, ", ftype: ", extra.get( 'ftype', 'auto' ), ", extra: ", extra #Post upload attribute editing edit_attributes = extra.get( 'edit_attributes', [] ) @@ -94,7 +99,7 @@ elem_hid = elem.get( 'hid' ) elem_index += 1 try: - self.verify_dataset_correctness( outfile, hid=elem_hid, maxseconds=testdef.maxseconds, attributes=attributes ) + self.verify_dataset_correctness( outfile, hid=elem_hid, maxseconds=testdef.maxseconds, attributes=attributes, migrated_tool_id=migrated_tool_id ) except Exception, e: print >>sys.stderr, self.get_job_stdout( elem.get( 'id' ), format=True ) print >>sys.stderr, self.get_job_stderr( elem.get( 'id' ), format=True ) @@ -138,7 +143,7 @@ expanded_inputs[value.name] = declared_inputs[value.name] return expanded_inputs -def build_tests(): +def build_tests( testing_migrated_tools=False ): """ If the module level variable `toolbox` is set, generate `ToolTestCase` classes for all of its tests and put them into this modules globals() so @@ -148,21 +153,30 @@ return # Push all the toolbox tests to module level G = globals() + # Eliminate all previous tests from G. + for key, val in G.items(): + if key.startswith( 'TestForTool_' ): + del G[ key ] for i, tool_id in enumerate( toolbox.tools_by_id ): tool = toolbox.get_tool( tool_id ) if tool.tests: - # Create a new subclass of ToolTestCase dynamically adding methods - # names test_tool_XXX that run each test defined in the tool. - n = "TestForTool_" + tool.id.replace( ' ', '_' ) - s = ( ToolTestCase, ) - d = dict() + # Create a new subclass of ToolTestCase, dynamically adding methods + # named test_tool_XXX that run each test defined in the tool config. + name = "TestForTool_" + tool.id.replace( ' ', '_' ) + baseclasses = ( ToolTestCase, ) + namespace = dict() for j, testdef in enumerate( tool.tests ): - def make_test_method( td ): + def make_test_method( td, migrated_tool_id=None ): def test_tool( self ): - self.do_it( td ) + self.do_it( td, migrated_tool_id=migrated_tool_id ) return test_tool - m = make_test_method( testdef ) - m.__doc__ = "%s ( %s ) > %s" % ( tool.name, tool.id, testdef.name ) - d['test_tool_%06d' % j] = m - G[ n ] = new.classobj( n, s, d ) - + if testing_migrated_tools: + test_method = make_test_method( testdef, migrated_tool_id=tool.id ) + else: + test_method = make_test_method( testdef ) + test_method.__doc__ = "%s ( %s ) > %s" % ( tool.name, tool.id, testdef.name ) + namespace[ 'test_tool_%06d' % j ] = test_method + # The new.classobj function returns a new class object, with name name, derived + # from baseclasses (which should be a tuple of classes) and with namespace dict. + new_class_obj = new.classobj( name, baseclasses, namespace ) + G[ name ] = new_class_obj Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Add GenomeSpace tools.
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/7ab3012fe281/ changeset: 7ab3012fe281 user: dan date: 2012-03-29 16:24:28 summary: Add GenomeSpace tools. affected #: 13 files diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tool_conf.xml.sample --- a/tool_conf.xml.sample +++ b/tool_conf.xml.sample @@ -27,14 +27,14 @@ <tool file="data_source/epigraph_import.xml" /><tool file="data_source/epigraph_import_test.xml" /><tool file="data_source/hbvar.xml" /> - <tool file="data_source/genomespace_file_browser_prod.xml" /> -  + <tool file="genomespace/genomespace_file_browser_prod.xml" /> + <tool file="genomespace/genomespace_importer.xml" /><tool file="validation/fix_errors.xml" /></section><section name="Send Data" id="send"><tool file="data_destination/epigraph.xml" /><tool file="data_destination/epigraph_test.xml" /> + <tool file="genomespace/genomespace_exporter.xml" /></section><section name="ENCODE Tools" id="EncodeTools"><tool file="encode/gencode_partition.xml" /> diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/data_source/genomespace_file_browser.py --- a/tools/data_source/genomespace_file_browser.py +++ /dev/null @@ -1,148 +0,0 @@ -#Dan Blankenberg - -import optparse, os, urllib2, cookielib - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson - -GENOMESPACE_API_VERSION_STRING = "v1.0" -GENOMESPACE_SERVER_URL_PROPERTIES = "http://www.genomespace.org/sites/genomespacefiles/config/serverurl.properti…" - -CHUNK_SIZE = 2**20 #1mb - -DEFAULT_GALAXY_EXT = "data" - -#genomespace format identifier is the URL -GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT = {} #TODO: fix this so it is not a global variable -#TODO: we should use a better way to set up this mapping -GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles', - 'lifes': 'lifes', - 'cn': 'cn', - 'GTF': 'gtf', - 'res': 'res', - 'xcn': 'xcn', - 'lowercasetxt': 'lowercasetxt', - 'bed': 'bed', - 'CBS': 'cbs', - 'genomicatab': 'genomicatab', - 'gxp': 'gxp', - 'reversedtxt': 'reversedtxt', - 'nowhitespace': 'nowhitespace', - 'unknown': 'unknown', - 'txt': 'txt', - 'uppercasetxt': 'uppercasetxt', - 'GISTIC': 'gistic', - 'GFF': 'gff', - 'gmt': 'gmt', - 'gct': 'gct'} - - -def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ): - source_method = getattr( source_stream, source_method ) - target_method = getattr( target_stream, target_method ) - while True: - chunk = source_method( CHUNK_SIZE ) - if chunk: - target_method( chunk ) - else: - break - -def get_cookie_opener( gs_username, gs_token ): - """ Create a GenomeSpace cookie opener """ - cj = cookielib.CookieJar() - for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]: - #create a super-cookie, valid for all domains - cookie = cookielib.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False ) - cj.set_cookie( cookie ) - cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) ) - return cookie_opener - -def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url ): - ext = GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.get( file_format_url, None ) - if ext is not None: - ext = GENOMESPACE_EXT_TO_GALAXY_EXT.get( ext, None ) - if ext is None: - #could check content type, etc here - ext = DEFAULT_GALAXY_EXT - return ext - -def get_genomespace_site_urls(): - genomespace_sites = {} - for line in urllib2.urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ): - line = line.rstrip() - if not line or line.startswith( "#" ): - continue - server, line = line.split( '.', 1 ) - if server not in genomespace_sites: - genomespace_sites[server] = {} - line = line.split( "=", 1 ) - genomespace_sites[server][line[0]] = line[1] - return genomespace_sites - -def set_genomespace_format_identifiers( url_opener, dm_site ): - gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) ) - gs_request.get_method = lambda: 'GET' - opened_gs_request = url_opener.open( gs_request ) - genomespace_formats = simplejson.loads( opened_gs_request.read() ) - for format in genomespace_formats: - GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name'] - -def download_from_genomespace_file_browser( json_parameter_file, genomespace_site ): - json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) - datasource_params = json_params.get( 'param_dict' ) - username = datasource_params.get( "gs-username", None ) - token = datasource_params.get( "gs-token", None ) - assert None not in [ username, token ], "Missing GenomeSpace username or token." - output_filename = datasource_params.get( "output", None ) - dataset_id = json_params['output_data'][0]['dataset_id'] - hda_id = json_params['output_data'][0]['hda_id'] - url_opener = get_cookie_opener( username, token ) - #load and set genomespace format ids to galaxy exts - genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] - set_genomespace_format_identifiers( url_opener, genomespace_site_dict['dmServer'] ) - - file_url_prefix = "fileUrl" - file_type_prefix = "fileFormat" - metadata_parameter_file = open( json_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'wb' ) - file_numbers = [] - for name in datasource_params.keys(): - if name.startswith( file_url_prefix ): - name = name[len( file_url_prefix ):] - file_numbers.append( int( name ) ) - file_numbers.sort() - for file_num in file_numbers: - url_key = "%s%i" % ( file_url_prefix, file_num ) - download_url = datasource_params.get( url_key, None ) - if download_url is None: - break - filetype_key = "%s%i" % ( file_type_prefix, file_num ) - filetype_url = datasource_params.get( filetype_key, None ) - galaxy_ext = get_galaxy_ext_from_genomespace_format_url( url_opener, filetype_url ) - if output_filename is None: - output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%i_visible_%s' % ( hda_id, file_num, galaxy_ext ) ) - else: - if dataset_id is not None: - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', - dataset_id = dataset_id, - ext = galaxy_ext ) ) ) - output_file = open( output_filename, 'wb' ) - new_file_request = urllib2.Request( download_url ) - new_file_request.get_method = lambda: 'GET' - target_download_url = url_opener.open( new_file_request ) - chunk_write( target_download_url, output_file ) - output_file.close() - output_filename = None #only have one filename available - metadata_parameter_file.close() - return True - -if __name__ == '__main__': - #Parse Command Line - parser = optparse.OptionParser() - parser.add_option( '-p', '--json_parameter_file', dest='json_parameter_file', action='store', type="string", default=None, help='json_parameter_file' ) - parser.add_option( '-s', '--genomespace_site', dest='genomespace_site', action='store', type="string", default=None, help='genomespace_site' ) - (options, args) = parser.parse_args() - - download_from_genomespace_file_browser( options.json_parameter_file, options.genomespace_site ) diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/data_source/genomespace_file_browser_dev.xml --- a/tools/data_source/genomespace_file_browser_dev.xml +++ /dev/null @@ -1,15 +0,0 @@ -<?xml version="1.0"?> -<tool name="GenomeSpace import" id="genomespace_file_browser_dev" tool_type="data_source" add_galaxy_url="False" force_history_refresh="True" version="0.0.1"> - <description>from file browser (development)</description> - <command interpreter="python">genomespace_file_browser.py --json_parameter_file "${output}" --genomespace_site "dev"</command> - <inputs action="https://dmdev.genomespace.org:8444/datamanager/defaultdirectory" check_values="False" method="post"> - <display>go to GenomeSpace Data Manager </display> - <param name="appCallbackUrl" type="baseurl" value="/tool_runner?tool_id=genomespace_file_browser_dev&runtool_btn=Execute" /> - <param name="appName" type="hidden" value="Galaxy" /> - </inputs> - <uihints minwidth="800"/> - <outputs> - <data name="output" format="auto" /> - </outputs> - <options sanitize="False" refresh="True"/> -</tool> diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/data_source/genomespace_file_browser_prod.xml --- a/tools/data_source/genomespace_file_browser_prod.xml +++ /dev/null @@ -1,15 +0,0 @@ -<?xml version="1.0"?> -<tool name="GenomeSpace import" id="genomespace_file_browser_prod" tool_type="data_source" add_galaxy_url="False" force_history_refresh="True" version="0.0.1"> - <description>from file browser</description> - <command interpreter="python">genomespace_file_browser.py --json_parameter_file "${output}" --genomespace_site "prod"</command> - <inputs action="https://dm.genomespace.org/datamanager/defaultdirectory" check_values="False" method="post"> - <display>go to GenomeSpace Data Manager </display> - <param name="appCallbackUrl" type="baseurl" value="/tool_runner?tool_id=genomespace_file_browser_prod&runtool_btn=Execute" /> - <param name="appName" type="hidden" value="Galaxy" /> - </inputs> - <uihints minwidth="800"/> - <outputs> - <data name="output" format="auto" /> - </outputs> - <options sanitize="False" refresh="True"/> -</tool> diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/data_source/genomespace_file_browser_test.xml --- a/tools/data_source/genomespace_file_browser_test.xml +++ /dev/null @@ -1,15 +0,0 @@ -<?xml version="1.0"?> -<tool name="GenomeSpace import" id="genomespace_file_browser_test" tool_type="data_source" add_galaxy_url="False" force_history_refresh="True" version="0.0.1"> - <description>from file browser (test)</description> - <command interpreter="python">genomespace_file_browser.py --json_parameter_file "${output}" --genomespace_site "test"</command> - <inputs action="https://dmtest.genomespace.org:8444/datamanager/defaultdirectory" check_values="False" method="post"> - <display>go to GenomeSpace Data Manager </display> - <param name="appCallbackUrl" type="baseurl" value="/tool_runner?tool_id=genomespace_file_browser_test&runtool_btn=Execute" /> - <param name="appName" type="hidden" value="Galaxy" /> - </inputs> - <uihints minwidth="800"/> - <outputs> - <data name="output" format="auto" /> - </outputs> - <options sanitize="False" refresh="True"/> -</tool> diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/genomespace/genomespace_exporter.py --- /dev/null +++ b/tools/genomespace/genomespace_exporter.py @@ -0,0 +1,208 @@ +#Dan Blankenberg + +import optparse, os, urllib2, urllib, cookielib, hashlib, base64, cgi, binascii + +from galaxy import eggs +import pkg_resources + +pkg_resources.require( "simplejson" ) +import simplejson + +GENOMESPACE_API_VERSION_STRING = "v1.0" +GENOMESPACE_SERVER_URL_PROPERTIES = "http://www.genomespace.org/sites/genomespacefiles/config/serverurl.properti…" + +CHUNK_SIZE = 2**20 #1mb + + +def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ): + source_method = getattr( source_stream, source_method ) + target_method = getattr( target_stream, target_method ) + while True: + chunk = source_method( CHUNK_SIZE ) + if chunk: + target_method( chunk ) + else: + break + +def get_cookie_opener( gs_username, gs_token ): + """ Create a GenomeSpace cookie opener """ + cj = cookielib.CookieJar() + for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]: + #create a super-cookie, valid for all domains + cookie = cookielib.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False ) + cj.set_cookie( cookie ) + cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) ) + return cookie_opener + +def get_genomespace_site_urls(): + genomespace_sites = {} + for line in urllib2.urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ): + line = line.rstrip() + if not line or line.startswith( "#" ): + continue + server, line = line.split( '.', 1 ) + if server not in genomespace_sites: + genomespace_sites[server] = {} + line = line.split( "=", 1 ) + genomespace_sites[server][line[0]] = line[1] + return genomespace_sites + +def get_directory( url_opener, dm_url, path ): + url = dm_url + for sub_path in path: + url = "%s/%s" % ( url, sub_path ) + dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } ) + dir_request.get_method = lambda: 'GET' + try: + dir_dict = simplejson.loads( url_opener.open( dir_request ).read() ) + except urllib2.HTTPError, e: + #print "e", e, url #punting, assuming lack of permisions at this low of a level... + continue + break + return dir_dict + +def get_default_directory( url_opener, dm_url ): + return get_directory( url_opener, dm_url, ["defaultdirectory"] ) + +def create_directory( url_opener, directory_dict, new_dir, dm_url ): + payload = { "isDirectory": True } + for dir_slice in new_dir: + if dir_slice in ( '', '/', None ): + continue + url = '/'.join( ( directory_dict['url'], urllib.quote( dir_slice.replace( '/', '_' ), safe='' ) ) ) + new_dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' }, data = simplejson.dumps( payload ) ) + new_dir_request.get_method = lambda: 'PUT' + directory_dict = simplejson.loads( url_opener.open( new_dir_request ).read() ) + return directory_dict + +def get_genome_space_launch_apps( atm_url, url_opener, file_url, file_type ): + gs_request = urllib2.Request( "%s/%s/webtool/descriptor" % ( atm_url, GENOMESPACE_API_VERSION_STRING ) ) + gs_request.get_method = lambda: 'GET' + opened_gs_request = url_opener.open( gs_request ) + webtool_descriptors = simplejson.loads( opened_gs_request.read() ) + webtools = [] + for webtool in webtool_descriptors: + webtool_name = webtool.get( 'name' ) + base_url = webtool.get( 'baseUrl' ) + use_tool = False + for param in webtool.get( 'fileParameters', [] ): + for format in param.get( 'formats', [] ): + if format.get( 'name' ) == file_type: + use_tool = True + break + if use_tool: + file_param_name = param.get( 'name' ) + #file_name_delimiters = param.get( 'nameDelimiters' ) + if '?' in base_url: + url_delimiter = "&" + else: + url_delimiter = "?" + launch_url = "%s%s%s" % ( base_url, url_delimiter, urllib.urlencode( [ ( file_param_name, file_url ) ] ) ) + webtools.append( ( launch_url, webtool_name ) ) + break + return webtools + +def galaxy_code_get_genomespace_folders( genomespace_site='prod', trans=None, value=None, **kwd ): + if value: + value = value[0]#single select, only 1 value + def recurse_directory_dict( url_opener, cur_options, url ): + cur_directory = urllib2.Request( url )#, headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct + cur_directory.get_method = lambda: 'GET' + #get url to upload to + cur_directory = url_opener.open( cur_directory ).read() + cur_directory = simplejson.loads( cur_directory ) + directory = cur_directory.get( 'directory', {} ) + contents = cur_directory.get( 'contents', [] ) + if directory.get( 'isDirectory', False ): + selected = directory.get( 'path' ) == value + cur_options.append( { 'name':directory.get( 'name' ), 'value': directory.get( 'path'), 'options':[], 'selected': selected } ) + for sub_dir in contents: + if sub_dir.get( 'isDirectory', False ): + recurse_directory_dict( url_opener, cur_options[-1]['options'], sub_dir.get( 'url' ) ) + rval = [] + if trans and trans.user: + username = trans.user.preferences.get( 'genomespace_username', None ) + token = trans.user.preferences.get( 'genomespace_token', None ) + if None in ( username, token ): + return [] + url_opener = get_cookie_opener( username, token ) + genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] + dm_url = genomespace_site_dict['dmServer'] + #get default directory + directory_dict = get_default_directory( url_opener, dm_url )['directory'] + #what directory to stuff this in + recurse_directory_dict( url_opener, rval, directory_dict.get( 'url' ) ) + + return rval + + +def send_file_to_genomespace( genomespace_site, username, token, source_filename, target_directory, target_filename, file_type, content_type, log_filename ): + url_opener = get_cookie_opener( username, token ) + genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] + dm_url = genomespace_site_dict['dmServer'] + #get default directory + if target_directory and target_directory[0] == '/': + directory_dict = get_directory( url_opener, dm_url, [ "%s/%s/%s" % ( GENOMESPACE_API_VERSION_STRING, 'file', target_directory[1] ) ] + target_directory[2:] )['directory'] + target_directory.pop(0) + else: + directory_dict = get_default_directory( url_opener, dm_url )['directory'] + #what directory to stuff this in + target_directory_dict = create_directory( url_opener, directory_dict, target_directory, dm_url ) + #get upload url + upload_url = "uploadurl" + content_length = os.path.getsize( source_filename ) + input_file = open( source_filename ) + content_md5 = hashlib.md5() + chunk_write( input_file, content_md5, target_method="update" ) + input_file.seek( 0 ) #back to start, for uploading + + upload_params = { 'Content-Length': content_length, 'Content-MD5': base64.standard_b64encode( content_md5.digest() ), 'Content-Type': content_type } + upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ), urllib.urlencode( upload_params ) ) + new_file_request = urllib2.Request( upload_url )#, headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct + new_file_request.get_method = lambda: 'GET' + #get url to upload to + target_upload_url = url_opener.open( new_file_request ).read() + #upload file to determined url + upload_headers = dict( upload_params ) + #upload_headers[ 'x-amz-meta-md5-hash' ] = content_md5.hexdigest() + upload_headers[ 'Accept' ] = 'application/json' + upload_file_request = urllib2.Request( target_upload_url, headers = upload_headers, data = input_file ) + upload_file_request.get_method = lambda: 'PUT' + upload_result = urllib2.urlopen( upload_file_request ).read() + + result_url = "%s/%s" % ( target_directory_dict['url'], urllib.quote( target_filename, safe='' ) ) + #determine available gs launch apps + web_tools = get_genome_space_launch_apps( genomespace_site_dict['atmServer'], url_opener, result_url, file_type ) + if log_filename: + log_file = open( log_filename, 'wb' ) + log_file.write( "<html><head><title>File uploaded to GenomeSpace from Galaxy</title></head><body>\n" ) + log_file.write( '<p>Uploaded <a href="%s">%s/%s</a> to GenomeSpace.</p>\n' % ( result_url, target_directory_dict['path'], target_filename ) ) + if web_tools: + log_file.write( "<p>You may open this file directly in the following applications:</p>\n" ) + log_file.write( '<p><ul>\n' ) + for web_tool in web_tools: + log_file.write( '<li><a href="%s">%s</a></li>\n' % ( web_tool ) ) + log_file.write( '</p></ul>\n' ) + else: + log_file.write( '<p>There are no GenomeSpace applications available for file type: %s</p>\n' % ( file_type ) ) + log_file.write( "</body></html>\n" ) + return upload_result + +if __name__ == '__main__': + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-s', '--genomespace_site', dest='genomespace_site', action='store', type="string", default=None, help='genomespace_site' ) + parser.add_option( '-t', '--token', dest='token', action='store', type="string", default=None, help='token' ) + parser.add_option( '-u', '--username', dest='username', action='store', type="string", default=None, help='username' ) + parser.add_option( '-d', '--dataset', dest='dataset', action='store', type="string", default=None, help='dataset' ) + parser.add_option( '-f', '--filename', dest='filename', action='store', type="string", default=None, help='filename' ) + parser.add_option( '-y', '--subdirectory', dest='subdirectory', action='append', type="string", default=None, help='subdirectory' ) + parser.add_option( '', '--file_type', dest='file_type', action='store', type="string", default=None, help='file_type' ) + parser.add_option( '-c', '--content_type', dest='content_type', action='store', type="string", default=None, help='content_type' ) + parser.add_option( '-l', '--log', dest='log', action='store', type="string", default=None, help='log' ) + + (options, args) = parser.parse_args() + + send_file_to_genomespace( options.genomespace_site, options.username, options.token, options.dataset, map( binascii.unhexlify, options.subdirectory ), options.filename, options.file_type, options.content_type, options.log ) + + diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/genomespace/genomespace_exporter.xml --- /dev/null +++ b/tools/genomespace/genomespace_exporter.xml @@ -0,0 +1,51 @@ +<?xml version="1.0"?> +<tool name="GenomeSpace Exporter" id="genomespace_exporter" require_login="True" version="0.0.1"> + <description> - send data to GenomeSpace</description> + <command interpreter="python">genomespace_exporter.py + --genomespace_site "prod" + #assert $__user_id__ != 'Anonymous', Exception( 'You must be logged in to use this tool.' ) + #set $user = $__app__.model.User.get( $__user_id__ ) + #set $username = $user.preferences.get( 'genomespace_username', None ) + #set $token = $user.preferences.get( 'genomespace_token', None ) + #assert None not in ( $username, $token ), Exception( 'You must associate a GenomeSpace OpenID with your account and log in with it.' ) + #import binascii + --username "${username}" + --token "${token}" + --dataset "${input1}" + #if $subdirectory: + #for $subd in str( $subdirectory ).split( '/' ): + #if not $subd: + --subdirectory "${ binascii.hexlify( '/' ) }" + #else: + --subdirectory "${ binascii.hexlify( $subd ) }" + #end if + #end for + #else: + --subdirectory "${ binascii.hexlify( 'galaxy_export' ) }" + --subdirectory "${ binascii.hexlify( str( $base_url ).split( '://', 1 )[-1] ) }" ##Protocol removed by request + #end if + #if $filename: + --filename "${filename}" + #else: + --filename "Galaxy History Item ${__app__.security.encode_id( $input1.id )} - ${input1.hid}: ${input1.name}.${input1.ext}" + #end if + --file_type "${input1.ext}" + --content_type "${input1.get_mime()}" + --log "${output_log}" + </command> + <inputs> + <param format="data" name="input1" type="data" label="Send this dataset to GenomeSpace" /> + <param name="base_url" type="baseurl" /> +  + <param name="subdirectory" type="drill_down" display="radio" hierarchy="exact" multiple="False" label="Choose Target Directory" dynamic_options="galaxy_code_get_genomespace_folders( genomespace_site = 'prod', trans=__trans__, value=__value__, input_dataset=input1 )" help="Leave blank to generate automatically"/> + <param name="filename" type="text" size="80" help="Leave blank to generate automatically" /> + </inputs> + <outputs> + <data format="html" name="output_log" /> + </outputs> + <help> + This Tool allows you to export data to GenomeSpace. You must have logged in using your GenomeSpace OpenID. You can associate your OpenID credentials under the User Preferences panel. + </help> + <options refresh="True"/> + <code file="genomespace_exporter.py" /> +</tool> \ No newline at end of file diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/genomespace/genomespace_file_browser.py --- /dev/null +++ b/tools/genomespace/genomespace_file_browser.py @@ -0,0 +1,160 @@ +#Dan Blankenberg + +import optparse, os, urllib, urllib2, urlparse, cookielib + +from galaxy import eggs +import pkg_resources + +pkg_resources.require( "simplejson" ) +import simplejson + +GENOMESPACE_API_VERSION_STRING = "v1.0" +GENOMESPACE_SERVER_URL_PROPERTIES = "http://www.genomespace.org/sites/genomespacefiles/config/serverurl.properti…" + +CHUNK_SIZE = 2**20 #1mb + +DEFAULT_GALAXY_EXT = "data" + +#genomespace format identifier is the URL +GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT = {} #TODO: fix this so it is not a global variable +#TODO: we should use a better way to set up this mapping +GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles', + 'lifes': 'lifes', + 'cn': 'cn', + 'GTF': 'gtf', + 'res': 'res', + 'xcn': 'xcn', + 'lowercasetxt': 'lowercasetxt', + 'bed': 'bed', + 'CBS': 'cbs', + 'genomicatab': 'genomicatab', + 'gxp': 'gxp', + 'reversedtxt': 'reversedtxt', + 'nowhitespace': 'nowhitespace', + 'unknown': 'unknown', + 'txt': 'txt', + 'uppercasetxt': 'uppercasetxt', + 'GISTIC': 'gistic', + 'GFF': 'gff', + 'gmt': 'gmt', + 'gct': 'gct'} + + +def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ): + source_method = getattr( source_stream, source_method ) + target_method = getattr( target_stream, target_method ) + while True: + chunk = source_method( CHUNK_SIZE ) + if chunk: + target_method( chunk ) + else: + break + +def get_cookie_opener( gs_username, gs_token ): + """ Create a GenomeSpace cookie opener """ + cj = cookielib.CookieJar() + for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]: + #create a super-cookie, valid for all domains + cookie = cookielib.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False ) + cj.set_cookie( cookie ) + cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) ) + return cookie_opener + +def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url ): + ext = GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.get( file_format_url, None ) + if ext is not None: + ext = GENOMESPACE_EXT_TO_GALAXY_EXT.get( ext, None ) + if ext is None: + #could check content type, etc here + ext = DEFAULT_GALAXY_EXT + return ext + +def get_genomespace_site_urls(): + genomespace_sites = {} + for line in urllib2.urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ): + line = line.rstrip() + if not line or line.startswith( "#" ): + continue + server, line = line.split( '.', 1 ) + if server not in genomespace_sites: + genomespace_sites[server] = {} + line = line.split( "=", 1 ) + genomespace_sites[server][line[0]] = line[1] + return genomespace_sites + +def set_genomespace_format_identifiers( url_opener, dm_site ): + gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) ) + gs_request.get_method = lambda: 'GET' + opened_gs_request = url_opener.open( gs_request ) + genomespace_formats = simplejson.loads( opened_gs_request.read() ) + for format in genomespace_formats: + GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name'] + +def download_from_genomespace_file_browser( json_parameter_file, genomespace_site ): + json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) + datasource_params = json_params.get( 'param_dict' ) + username = datasource_params.get( "gs-username", None ) + token = datasource_params.get( "gs-token", None ) + assert None not in [ username, token ], "Missing GenomeSpace username or token." + output_filename = datasource_params.get( "output", None ) + dataset_id = json_params['output_data'][0]['dataset_id'] + hda_id = json_params['output_data'][0]['hda_id'] + url_opener = get_cookie_opener( username, token ) + #load and set genomespace format ids to galaxy exts + genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] + set_genomespace_format_identifiers( url_opener, genomespace_site_dict['dmServer'] ) + + file_url_prefix = "fileUrl" + file_type_prefix = "fileFormat" + metadata_parameter_file = open( json_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'wb' ) + file_numbers = [] + for name in datasource_params.keys(): + if name.startswith( file_url_prefix ): + name = name[len( file_url_prefix ):] + file_numbers.append( int( name ) ) + file_numbers.sort() + for file_num in file_numbers: + url_key = "%s%i" % ( file_url_prefix, file_num ) + download_url = datasource_params.get( url_key, None ) + if download_url is None: + break + filetype_key = "%s%i" % ( file_type_prefix, file_num ) + filetype_url = datasource_params.get( filetype_key, None ) + galaxy_ext = get_galaxy_ext_from_genomespace_format_url( url_opener, filetype_url ) + formated_download_url = "%s?%s" % ( download_url, urllib.urlencode( [ ( 'dataformat', filetype_url ) ] ) ) + new_file_request = urllib2.Request( formated_download_url ) + new_file_request.get_method = lambda: 'GET' + target_download_url = url_opener.open( new_file_request ) + filename = None + if 'Content-Disposition' in target_download_url.info(): + # If the response has Content-Disposition, try to get filename from it + content_disposition = dict( map( lambda x: x.strip().split('=') if '=' in x else ( x.strip(),'' ), target_download_url.info()['Content-Disposition'].split( ';' ) ) ) + if 'filename' in content_disposition: + filename = content_disposition[ 'filename' ].strip( "\"'" ) + if not filename: + parsed_url = urlparse.urlparse( download_url ) + query_params = urlparse.parse_qs( parsed_url[4] ) + filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] ) + if output_filename is None: + output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%i_visible_%s' % ( hda_id, file_num, galaxy_ext ) ) + else: + if dataset_id is not None: + metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', + dataset_id = dataset_id, + ext = galaxy_ext, + name = "GenomeSpace import on %s" % ( filename ) ) ) ) + output_file = open( output_filename, 'wb' ) + chunk_write( target_download_url, output_file ) + output_file.close() + output_filename = None #only have one filename available + metadata_parameter_file.close() + return True + +if __name__ == '__main__': + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-p', '--json_parameter_file', dest='json_parameter_file', action='store', type="string", default=None, help='json_parameter_file' ) + parser.add_option( '-s', '--genomespace_site', dest='genomespace_site', action='store', type="string", default=None, help='genomespace_site' ) + (options, args) = parser.parse_args() + + download_from_genomespace_file_browser( options.json_parameter_file, options.genomespace_site ) diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/genomespace/genomespace_file_browser_dev.xml --- /dev/null +++ b/tools/genomespace/genomespace_file_browser_dev.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<tool name="GenomeSpace import" id="genomespace_file_browser_dev" tool_type="data_source" add_galaxy_url="False" force_history_refresh="True" version="0.0.1"> + <description>from file browser (development)</description> + <command interpreter="python">genomespace_file_browser.py --json_parameter_file "${output}" --genomespace_site "dev"</command> + <inputs action="https://dmdev.genomespace.org:8444/datamanager/defaultdirectory" check_values="False" method="post"> + <display>go to GenomeSpace Data Manager </display> + <param name="appCallbackUrl" type="baseurl" value="/tool_runner?tool_id=genomespace_file_browser_dev&runtool_btn=Execute" /> + <param name="appName" type="hidden" value="Galaxy" /> + </inputs> + <uihints minwidth="800"/> + <outputs> + <data name="output" format="auto" /> + </outputs> + <options sanitize="False" refresh="True"/> +</tool> diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/genomespace/genomespace_file_browser_prod.xml --- /dev/null +++ b/tools/genomespace/genomespace_file_browser_prod.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<tool name="GenomeSpace import" id="genomespace_file_browser_prod" tool_type="data_source" add_galaxy_url="False" force_history_refresh="True" version="0.0.1"> + <description>from file browser</description> + <command interpreter="python">genomespace_file_browser.py --json_parameter_file "${output}" --genomespace_site "prod"</command> + <inputs action="https://dm.genomespace.org/datamanager/defaultdirectory" check_values="False" method="post"> + <display>go to GenomeSpace Data Manager </display> + <param name="appCallbackUrl" type="baseurl" value="/tool_runner?tool_id=genomespace_file_browser_prod&runtool_btn=Execute" /> + <param name="appName" type="hidden" value="Galaxy" /> + </inputs> + <uihints minwidth="800"/> + <outputs> + <data name="output" format="auto" /> + </outputs> + <options sanitize="False" refresh="True"/> +</tool> diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/genomespace/genomespace_file_browser_test.xml --- /dev/null +++ b/tools/genomespace/genomespace_file_browser_test.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<tool name="GenomeSpace import" id="genomespace_file_browser_test" tool_type="data_source" add_galaxy_url="False" force_history_refresh="True" version="0.0.1"> + <description>from file browser (test)</description> + <command interpreter="python">genomespace_file_browser.py --json_parameter_file "${output}" --genomespace_site "test"</command> + <inputs action="https://dmtest.genomespace.org:8444/datamanager/defaultdirectory" check_values="False" method="post"> + <display>go to GenomeSpace Data Manager </display> + <param name="appCallbackUrl" type="baseurl" value="/tool_runner?tool_id=genomespace_file_browser_test&runtool_btn=Execute" /> + <param name="appName" type="hidden" value="Galaxy" /> + </inputs> + <uihints minwidth="800"/> + <outputs> + <data name="output" format="auto" /> + </outputs> + <options sanitize="False" refresh="True"/> +</tool> diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/genomespace/genomespace_importer.py --- /dev/null +++ b/tools/genomespace/genomespace_importer.py @@ -0,0 +1,156 @@ +#Dan Blankenberg + +import optparse, os, urllib2, urllib, cookielib, urlparse + +from galaxy import eggs +import pkg_resources + +pkg_resources.require( "simplejson" ) +import simplejson + +GENOMESPACE_API_VERSION_STRING = "v1.0" +GENOMESPACE_SERVER_URL_PROPERTIES = "http://www.genomespace.org/sites/genomespacefiles/config/serverurl.properti…" + +CHUNK_SIZE = 2**20 #1mb + +DEFAULT_GALAXY_EXT = "data" + +#genomespace format identifier is the URL +GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT = {} #TODO: fix this so it is not a global variable +#TODO: we should use a better way to set up this mapping +GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles', + 'lifes': 'lifes', + 'cn': 'cn', + 'GTF': 'gtf', + 'res': 'res', + 'xcn': 'xcn', + 'lowercasetxt': 'lowercasetxt', + 'bed': 'bed', + 'CBS': 'cbs', + 'genomicatab': 'genomicatab', + 'gxp': 'gxp', + 'reversedtxt': 'reversedtxt', + 'nowhitespace': 'nowhitespace', + 'unknown': 'unknown', + 'txt': 'txt', + 'uppercasetxt': 'uppercasetxt', + 'GISTIC': 'gistic', + 'GFF': 'gff', + 'gmt': 'gmt', + 'gct': 'gct'} + +VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + +def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ): + source_method = getattr( source_stream, source_method ) + target_method = getattr( target_stream, target_method ) + while True: + chunk = source_method( CHUNK_SIZE ) + if chunk: + target_method( chunk ) + else: + break + +def get_cookie_opener( gs_username, gs_token ): + """ Create a GenomeSpace cookie opener """ + cj = cookielib.CookieJar() + for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]: + #create a super-cookie, valid for all domains + cookie = cookielib.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False ) + cj.set_cookie( cookie ) + cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) ) + return cookie_opener + +def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url ): + ext = GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.get( file_format_url, None ) + if ext is not None: + ext = GENOMESPACE_EXT_TO_GALAXY_EXT.get( ext, None ) + if ext is None: + #could check content type, etc here + ext = DEFAULT_GALAXY_EXT + return ext + +def get_genomespace_site_urls(): + genomespace_sites = {} + for line in urllib2.urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ): + line = line.rstrip() + if not line or line.startswith( "#" ): + continue + server, line = line.split( '.', 1 ) + if server not in genomespace_sites: + genomespace_sites[server] = {} + line = line.split( "=", 1 ) + genomespace_sites[server][line[0]] = line[1] + return genomespace_sites + +def set_genomespace_format_identifiers( url_opener, dm_site ): + gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) ) + gs_request.get_method = lambda: 'GET' + opened_gs_request = url_opener.open( gs_request ) + genomespace_formats = simplejson.loads( opened_gs_request.read() ) + for format in genomespace_formats: + GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name'] + +def download_from_genomespace_importer( username, token, json_parameter_file, genomespace_site ): + json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) + datasource_params = json_params.get( 'param_dict' ) + #username = datasource_params.get( "gs-username", None ) + #token = datasource_params.get( "gs-token", None ) + assert None not in [ username, token ], "Missing GenomeSpace username or token." + output_filename = datasource_params.get( "output_file1", None ) + dataset_id = json_params['output_data'][0]['dataset_id'] + hda_id = json_params['output_data'][0]['hda_id'] + url_opener = get_cookie_opener( username, token ) + #load and set genomespace format ids to galaxy exts + genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] + set_genomespace_format_identifiers( url_opener, genomespace_site_dict['dmServer'] ) + file_url_name = "URL" + metadata_parameter_file = open( json_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'wb' ) + url_param = datasource_params.get( file_url_name, None ) + for download_url in url_param.split( ',' ): + parsed_url = urlparse.urlparse( download_url ) + query_params = urlparse.parse_qs( parsed_url[4] ) + file_type = DEFAULT_GALAXY_EXT + if 'dataformat' in query_params: + file_type = query_params[ 'dataformat' ][0] + file_type = get_galaxy_ext_from_genomespace_format_url( url_opener, file_type ) + elif '.' in parsed_url[2]: + file_type = parsed_url[2].rsplit( '.', 1 )[-1] + file_type = GENOMESPACE_EXT_TO_GALAXY_EXT.get( file_type, file_type ) + new_file_request = urllib2.Request( download_url ) + new_file_request.get_method = lambda: 'GET' + target_download_url = url_opener.open( new_file_request ) + filename = None + if 'Content-Disposition' in target_download_url.info(): + content_disposition = dict( map( lambda x: x.strip().split('=') if '=' in x else ( x.strip(),'' ), target_download_url.info()['Content-Disposition'].split( ';' ) ) ) + if 'filename' in content_disposition: + filename = content_disposition[ 'filename' ].strip( "\"'" ) + if not filename: + parsed_url = urlparse.urlparse( download_url ) + query_params = urlparse.parse_qs( parsed_url[4] ) + filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] ) + if output_filename is None: + output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%s_visible_%s' % ( hda_id, ''.join( c in VALID_CHARS and c or '-' for c in filename ), file_type ) ) + else: + if dataset_id is not None: + metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', + dataset_id = dataset_id, + ext = file_type, + name = "GenomeSpace importer on %s" % ( filename ) ) ) ) + output_file = open( output_filename, 'wb' ) + chunk_write( target_download_url, output_file ) + output_file.close() + output_filename = None #only have one filename available + metadata_parameter_file.close() + return True + +if __name__ == '__main__': + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-p', '--json_parameter_file', dest='json_parameter_file', action='store', type="string", default=None, help='json_parameter_file' ) + parser.add_option( '-s', '--genomespace_site', dest='genomespace_site', action='store', type="string", default=None, help='genomespace_site' ) + parser.add_option( '-t', '--token', dest='token', action='store', type="string", default=None, help='token' ) + parser.add_option( '-u', '--username', dest='username', action='store', type="string", default=None, help='username' ) + (options, args) = parser.parse_args() + + download_from_genomespace_importer( options.username, options.token, options.json_parameter_file, options.genomespace_site ) diff -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f -r 7ab3012fe281f9219e2323e6c8c9694fc2b2628a tools/genomespace/genomespace_importer.xml --- /dev/null +++ b/tools/genomespace/genomespace_importer.xml @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<tool name="GenomeSpace Importer" id="genomespace_importer" tool_type="data_source" force_history_refresh="True" hidden="True" display_interface="False" require_login="True" version="0.0.1"> + <description> - receive data from GenomeSpace</description> + <command interpreter="python">genomespace_importer.py + --genomespace_site "prod" + #assert $__user_id__ != 'Anonymous', Exception( 'You must be logged in to use this tool.' ) + #set $user = $__app__.model.User.get( $__user_id__ ) + #set $username = $user.preferences.get( 'genomespace_username', None ) + #set $token = $user.preferences.get( 'genomespace_token', None ) + #assert None not in ( $username, $token ), Exception( 'You must associate a GenomeSpace OpenID with your account and log in with it.' ) + --username "${username}" + --token "${token}" + --json_parameter_file "${output_file1}" + </command> + <inputs check_values="False"> +  + <param name="URL" type="hidden" value="" /> + </inputs> + <outputs> + <data format="auto" name="output_file1" /> + </outputs> + <help> + some help text here... + </help> + <options refresh="True"/> +</tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Add require_login and display_interface flags to tools. Add 'everything' target for refresh_frame.
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/a2cd73dd4f72/ changeset: a2cd73dd4f72 user: dan date: 2012-03-29 16:24:15 summary: Add require_login and display_interface flags to tools. Add 'everything' target for refresh_frame. affected #: 3 files diff -r 5aab033197616ac6d8ca1f14f672f9f3195a56dc -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -636,6 +636,8 @@ self.check_values = True self.nginx_upload = False self.input_required = False + self.display_interface = True + self.require_login = False # Define a place to keep track of all input parameters. These # differ from the inputs dictionary in that inputs can be page # elements like conditionals, but input_params are basic form @@ -732,6 +734,8 @@ # Useful i.e. when an indeterminate number of outputs are created by # a tool. self.force_history_refresh = util.string_as_bool( root.get( 'force_history_refresh', 'False' ) ) + self.display_interface = util.string_as_bool( root.get( 'display_interface', str( self.display_interface ) ) ) + self.require_login = util.string_as_bool( root.get( 'require_login', str( self.require_login ) ) ) # Load input translator, used by datasource tools to change # names/values of incoming parameters self.input_translator = root.find( "request_param_translation" ) @@ -1337,6 +1341,8 @@ # on the standard run form) or "URL" (a parameter provided by # external data source tools). if "runtool_btn" not in incoming and "URL" not in incoming: + if not self.display_interface: + return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) return "tool_form.mako", dict( errors={}, tool_state=state, param_values={}, incoming={} ) # Process incoming data if not( self.check_values ): @@ -1385,6 +1391,8 @@ state.page += 1 # Fill in the default values for the next page self.fill_in_new_state( trans, self.inputs_by_page[ state.page ], state.inputs ) + if not self.display_interface: + return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) return 'tool_form.mako', dict( errors=errors, tool_state=state ) else: try: @@ -1397,6 +1405,8 @@ except: pass # Just a refresh, render the form with updated state and errors. + if not self.display_interface: + return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) return 'tool_form.mako', dict( errors=errors, tool_state=state ) def find_fieldstorage( self, x ): if isinstance( x, FieldStorage ): diff -r 5aab033197616ac6d8ca1f14f672f9f3195a56dc -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f lib/galaxy/web/controllers/tool_runner.py --- a/lib/galaxy/web/controllers/tool_runner.py +++ b/lib/galaxy/web/controllers/tool_runner.py @@ -58,6 +58,8 @@ log.error( "index called with tool id '%s' but no such tool exists", tool_id ) trans.log_event( "Tool id '%s' does not exist" % tool_id ) return "Tool '%s' does not exist, kwd=%s " % (tool_id, kwd) + if tool.require_login and not trans.user: + return trans.response.send_redirect( url_for( controller='user', action='login', cntrller='user', message="You must be logged in to use this tool.", status="info", referer=url_for( controller='/tool_runner', action='index', tool_id=tool_id, **kwd ) ) ) params = util.Params( kwd, sanitize = False ) #Sanitize parameters when substituting into command line via input wrappers #do param translation here, used by datasource tools if tool.input_translator: diff -r 5aab033197616ac6d8ca1f14f672f9f3195a56dc -r a2cd73dd4f7284b1d0ffa56235e478d013cc116f templates/message.mako --- a/templates/message.mako +++ b/templates/message.mako @@ -24,6 +24,9 @@ <%def name="javascripts()"> ${parent.javascripts()} <script type="text/javascript"> + %if 'everything' in refresh_frames: + parent.location.href="${h.url_for( controller='root' )}"; + %endif %if 'masthead' in refresh_frames: ## if ( parent.frames && parent.frames.galaxy_masthead ) { ## parent.frames.galaxy_masthead.location.href="${h.url_for( controller='root', action='masthead')}"; Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Enable dynamic options for DrillDownSelectToolParameter. Fix for setting checked state for current value for DrillDownSelectToolParameter.
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/5aab03319761/ changeset: 5aab03319761 user: dan date: 2012-03-29 16:23:42 summary: Enable dynamic options for DrillDownSelectToolParameter. Fix for setting checked state for current value for DrillDownSelectToolParameter. affected #: 2 files diff -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d -r 5aab033197616ac6d8ca1f14f672f9f3195a56dc lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1133,7 +1133,9 @@ from_file = os.path.join( tool.app.config.tool_data_path, from_file ) elem = XML( "<root>%s</root>" % open( from_file ).read() ) self.is_dynamic = False - self.dynamic_options = None #backwards compatibility with SelectToolParameter's old dynamic options and late validation + self.dynamic_options = elem.get( 'dynamic_options' , None ) + if self.dynamic_options: + self.is_dynamic = True self.options = [] self.filtered = {} if elem.find( 'filter' ): @@ -1148,12 +1150,23 @@ if filter.get( 'value' ) not in self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )]: self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )][filter.get( 'value' )] = [] recurse_option_elems( self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )][filter.get( 'value' )], filter.find( 'options' ).findall( 'option' ) ) - else: + elif not self.dynamic_options: recurse_option_elems( self.options, elem.find( 'options' ).findall( 'option' ) ) + def _get_options_from_code( self, trans=None, value=None, other_values=None ): + assert self.dynamic_options, Exception( "dynamic_options was not specifed" ) + call_other_values = { '__trans__': trans, '__value__': value } + if other_values: + call_other_values.update( other_values.dict ) + return eval( self.dynamic_options, self.tool.code_namespace, call_other_values ) + + def get_options( self, trans=None, value=None, other_values={} ): if self.is_dynamic: - options = [] + if self.dynamic_options: + options = self._get_options_from_code( trans=trans, value=value, other_values=other_values ) + else: + options = [] for filter_key, filter_value in self.filtered.iteritems(): dataset = other_values[filter_key] if dataset.__class__.__name__.endswith( "DatasetFilenameWrapper" ): #this is a bad way to check for this, but problems importing class ( due to circular imports? ) diff -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d -r 5aab033197616ac6d8ca1f14f672f9f3195a56dc lib/galaxy/web/form_builder.py --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -440,7 +440,7 @@ self.options = options if value and not isinstance( value, list ): value = [ value ] - else: + elif not value: value = [] self.value = value if display == "checkbox": Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fixes for transferring sample datasets and viewing . managing transferred sample datasets.
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/7273e1405094/ changeset: 7273e1405094 user: greg date: 2012-03-29 16:17:51 summary: Fixes for transferring sample datasets and viewing . managing transferred sample datasets. affected #: 7 files diff -r 9f5750e19c1ea88fa0c694bb42642589659b0750 -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d lib/galaxy/web/controllers/requests_admin.py --- a/lib/galaxy/web/controllers/requests_admin.py +++ b/lib/galaxy/web/controllers/requests_admin.py @@ -493,6 +493,7 @@ return trans.response.send_redirect( web.url_for( controller='requests_admin', action='select_datasets_to_transfer', request_id=trans.security.encode_id( request.id ), + external_service_id=trans.security.encode_id( external_service.id ), status=status, message=message ) ) def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): diff -r 9f5750e19c1ea88fa0c694bb42642589659b0750 -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d lib/galaxy/web/controllers/requests_common.py --- a/lib/galaxy/web/controllers/requests_common.py +++ b/lib/galaxy/web/controllers/requests_common.py @@ -1011,6 +1011,8 @@ sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) except: return invalid_id_redirect( trans, cntrller, sample_id, 'sample' ) + external_service_id = params.get( 'external_service_id', None ) + external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) # See if a library and folder have been set for this sample. if is_admin and not sample.library or not sample.folder: status = 'error' @@ -1043,6 +1045,7 @@ return trans.fill_template( '/requests/common/view_sample_datasets.mako', cntrller=cntrller, title=title, + external_service=external_service, sample=sample, sample_datasets=sample_datasets, transfer_status=transfer_status, diff -r 9f5750e19c1ea88fa0c694bb42642589659b0750 -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d templates/requests/common/common.mako --- a/templates/requests/common/common.mako +++ b/templates/requests/common/common.mako @@ -298,9 +298,14 @@ ## This link will direct the admin to a page allowing them to manage datasets. <a id="sampleDatasets-${sample.id}" href="${h.url_for( controller='requests_admin', action='manage_datasets', cntrller=cntrller, sample_id=trans.security.encode_id( sample.id ) )}">${len( sample.datasets )}</a> %elif sample.datasets: + <% + # Get an external_service from one of the sample datasets. This assumes all sample datasets are associated with + # the same external service - hopefully this is a good assumption. + external_service = sample.datasets[0].external_service + %> ## Since this is a regular user, only display a link if there is at least 1 ## selected dataset for the sample. - <a id="sampleDatasets-${sample.id}" href="${h.url_for( controller='requests_common', action='view_sample_datasets', cntrller=cntrller, sample_id=trans.security.encode_id( sample.id ) )}">${len( sample.datasets )}</a> + <a id="sampleDatasets-${sample.id}" href="${h.url_for( controller='requests_common', action='view_sample_datasets', cntrller=cntrller, external_service_id=trans.security.encode_id( external_service.id ), sample_id=trans.security.encode_id( sample.id ) )}">${len( sample.datasets )}</a> %else: ## Since this is a regular user, do not display a link if there are no datasets. <a id="sampleDatasets-${sample.id}">${len( sample.datasets )}</a> @@ -424,8 +429,13 @@ %endif %if sample.datasets and len( sample.datasets ) > len( transferred_dataset_files ) and sample.library and sample.folder: <li><a class="action-button" href="${h.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ) )}">Manage selected datasets</a></li> - %elif sample.datasets and len(sample.datasets ) == len( transferred_dataset_files ): - <li><a class="action-button" href="${h.url_for( controller='requests_common', action='view_sample_datasets', cntrller=cntrller, sample_id=trans.security.encode_id( sample.id ), transfer_status=trans.model.SampleDataset.transfer_status.COMPLETE )}">View transferred datasets</a></li> + %elif sample.datasets and len( sample.datasets ) == len( transferred_dataset_files ): + <% + # Get an external_service from one of the sample datasets. This assumes all sample datasets are associated with + # the same external service - hopefully this is a good assumption. + external_service = sample.datasets[0].external_service + %> + <li><a class="action-button" href="${h.url_for( controller='requests_common', action='view_sample_datasets', cntrller=cntrller, external_service_id=trans.security.encode_id( external_service.id ), sample_id=trans.security.encode_id( sample.id ), transfer_status=trans.model.SampleDataset.transfer_status.COMPLETE )}">View transferred datasets</a></li> %endif </div> %else: @@ -483,6 +493,11 @@ ## This link will direct the admin to a page allowing them to manage datasets. <a id="sampleDatasets-${sample.id}" href="${h.url_for( controller='requests_admin', action='manage_datasets', cntrller=cntrller, sample_id=trans.security.encode_id( sample.id ) )}">${len( sample.datasets )}</a> %elif sample.library and sample.datasets: + <% + # Get an external_service from one of the sample datasets. This assumes all sample datasets are associated with + # the same external service - hopefully this is a good assumption. + external_service = sample.datasets[0].external_service + %> ## Since this is a regular user, only display a link if there is at least 1 ## selected dataset for the sample. <a id="sampleDatasets-${sample.id}" href="${h.url_for( controller='requests_common', action='view_sample_datasets', cntrller=cntrller, sample_id=trans.security.encode_id( sample.id ) )}">${len( sample.datasets )}</a> diff -r 9f5750e19c1ea88fa0c694bb42642589659b0750 -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d templates/requests/common/find_samples.mako --- a/templates/requests/common/find_samples.mako +++ b/templates/requests/common/find_samples.mako @@ -78,7 +78,12 @@ %else: State: ${sample.state.name}<br/> %endif - Datasets: <a href="${h.url_for( controller='requests_common', action='view_sample_datasets', cntrller=cntrller, sample_id=trans.security.encode_id( sample.id ) )}">${len( sample.datasets )}</a><br/> + <% + # Get an external_service from one of the sample datasets. This assumes all sample datasets are associated with + # the same external service - hopefully this is a good assumption. + external_service = sample.datasets[0].external_service + %> + Datasets: <a href="${h.url_for( controller='requests_common', action='view_sample_datasets', cntrller=cntrller, external_service_id=trans.security.encode_id( external_service.id ), sample_id=trans.security.encode_id( sample.id ) )}">${len( sample.datasets )}</a><br/> %if is_admin: <i>User: ${sample.request.user.email}</i> %endif diff -r 9f5750e19c1ea88fa0c694bb42642589659b0750 -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d templates/requests/common/view_sample_datasets.mako --- a/templates/requests/common/view_sample_datasets.mako +++ b/templates/requests/common/view_sample_datasets.mako @@ -25,7 +25,7 @@ <li><a class="action-button" id="sample-${sample.id}-popup" class="menubutton">Dataset Actions</a></li><div popupmenu="sample-${sample.id}-popup"> %if can_select_datasets: - <li><a class="action-button" href="${h.url_for( controller='requests_admin', action='select_datasets_to_transfer', cntrller=cntrller, request_id=trans.security.encode_id( sample.request.id ), sample_id=trans.security.encode_id( sample.id ) )}">Select more datasets</a></li> + <li><a class="action-button" href="${h.url_for( controller='requests_admin', action='select_datasets_to_transfer', cntrller=cntrller, external_service_id=trans.security.encode_id( external_service.id ), request_id=trans.security.encode_id( sample.request.id ), sample_id=trans.security.encode_id( sample.id ) )}">Select more datasets</a></li> %endif <li><a class="action-button" href="${h.url_for( controller='library_common', action='browse_library', cntrller=cntrller, id=trans.security.encode_id( sample.library.id ) )}">View target Data Library</a></li><li><a class="action-button" href="${h.url_for( controller='requests_common', action='view_request', cntrller=cntrller, id=trans.security.encode_id( sample.request.id ) )}">Browse this request</a></li> diff -r 9f5750e19c1ea88fa0c694bb42642589659b0750 -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -13,6 +13,7 @@ from elementtree import ElementTree from galaxy.web import security from galaxy.web.framework.helpers import iff +from galaxy.util.json import from_json_string from base.asserts import verify_assertions buffer = StringIO.StringIO() @@ -34,7 +35,14 @@ self.host = os.environ.get( 'GALAXY_TEST_HOST' ) self.port = os.environ.get( 'GALAXY_TEST_PORT' ) self.url = "http://%s:%s" % ( self.host, self.port ) - self.file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR' ) + self.file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', None ) + self.migrated_tools_file = os.environ.get( 'GALAXY_MIGRATED_TOOLS_FILE', None ) + if self.migrated_tools_file: + f = open( self.migrated_tools_file, 'r' ) + self.migrated_tools_dict = from_json_string( f.readlines() ) + f.close() + else: + self.migrated_tools_dict = {} self.keepOutdir = os.environ.get( 'GALAXY_TEST_SAVE', '' ) if self.keepOutdir > '': try: @@ -1792,9 +1800,10 @@ tc.submit( "save_samples_button" ) for check_str in strings_displayed_after_submit: self.check_page_for_string( check_str ) - def add_datasets_to_sample( self, request_id, sample_id, sample_datasets, strings_displayed=[], strings_displayed_after_submit=[] ): + def add_datasets_to_sample( self, request_id, sample_id, external_service_id, sample_datasets, strings_displayed=[], strings_displayed_after_submit=[] ): # visit the dataset selection page - url = "%s/requests_admin/select_datasets_to_transfer?cntrller=requests_admin&sample_id=%s&request_id=%s" % ( self.url, sample_id, request_id ) + url = "%s/requests_admin/select_datasets_to_transfer?cntrller=requests_admin&sample_id=%s&request_id=%s&external_service_id=%s" % \ + ( self.url, sample_id, request_id, external_service_id ) self.visit_url( url ) for check_str in strings_displayed: self.check_page_for_string( check_str ) diff -r 9f5750e19c1ea88fa0c694bb42642589659b0750 -r 7273e1405094e931b4dc39f646cf4cd83a5bbc5d test/functional/test_sample_tracking.py --- a/test/functional/test_sample_tracking.py +++ b/test/functional/test_sample_tracking.py @@ -800,11 +800,13 @@ sample_dataset_file_names = [ dataset.split( '/' )[-1] for dataset in sample_datasets ] global request1_sample1 request1_sample1 = request1.get_sample( 'Sample1_renamed' ) + external_service = request1_sample1.external_service strings_displayed_after_submit = [ 'Datasets (%s) have been selected for sample (%s)' % \ ( str( sample_dataset_file_names )[1:-1].replace( "'", "" ), request1_sample1.name ) ] strings_displayed = [ 'Select datasets to transfer from data directory configured for the sequencer' ] self.add_datasets_to_sample( request_id=self.security.encode_id( request2.id ), sample_id= self.security.encode_id( request1_sample1.id ), + external_service_id=self.security.encode_id( external_serviceexternal_service.id ), sample_datasets=sample_datasets, strings_displayed=strings_displayed, strings_displayed_after_submit=strings_displayed_after_submit ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for managing sample datasets in the sample tracking admin UI.
by Bitbucket 29 Mar '12

29 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/9f5750e19c1e/ changeset: 9f5750e19c1e user: greg date: 2012-03-29 15:17:34 summary: Fix for managing sample datasets in the sample tracking admin UI. affected #: 1 file diff -r 04ed074ef9d21888bfd68235707b5ec0b9a70a10 -r 9f5750e19c1ea88fa0c694bb42642589659b0750 lib/galaxy/web/controllers/requests_admin.py --- a/lib/galaxy/web/controllers/requests_admin.py +++ b/lib/galaxy/web/controllers/requests_admin.py @@ -199,6 +199,11 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) + sample_id = params.get( 'sample_id', None ) + try: + sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id ( sample_id ) ) + except: + return invalid_id_redirect( trans, 'requests_admin', sample_id, 'sample' ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() sample_dataset_id = params.get( 'id', None ) @@ -269,11 +274,6 @@ **kwd ) ) # Render the grid view - sample_id = params.get( 'sample_id', None ) - try: - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id ( sample_id ) ) - except: - return invalid_id_redirect( trans, 'requests_admin', sample_id, 'sample' ) request_id = trans.security.encode_id( sample.request.id ) library_id = trans.security.encode_id( sample.library.id ) self.datatx_grid.title = 'Manage "%s" datasets' % sample.name Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Make ldda.templates_json able to set use_name flag in call to templates_dict.
by Bitbucket 28 Mar '12

28 Mar '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/04ed074ef9d2/ changeset: 04ed074ef9d2 user: dan date: 2012-03-28 18:05:00 summary: Make ldda.templates_json able to set use_name flag in call to templates_dict. affected #: 1 file diff -r 0c6cf4f53c4efad46fe7716587ca5a2dddff049c -r 04ed074ef9d21888bfd68235707b5ec0b9a70a10 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1554,8 +1554,8 @@ tmp_dict[ name ] = content[ field[ 'name' ] ] template_data[template.name] = tmp_dict return template_data - def templates_json( self ): - return simplejson.dumps( self.templates_dict() ) + def templates_json( self, use_name=False ): + return simplejson.dumps( self.templates_dict( use_name=use_name ) ) def get_display_name( self ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0