5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3f05316ce541/ Changeset: 3f05316ce541 Branch: fretn/fix-for-two-issues-the-secondary-user-1385478183593 User: fretn Date: 2013-11-26 16:04:57 Summary: Fix for two issues - The secondary user groups are not assigned to the user - When the json file is on a NFS share where root access is not allowed this script fails with a "error: JobTemplate file (/path/to/jsonfile) doesn't exist" error To fix the first we have to go through all the groups of the user in set_user(uid) and assign them with os.setgroups() The second one is fixed by removing the check if the json file exists in the function validate_paramters and move that check to a new function which is called after set_user(uid). Because its possible that the new user (the one in uid) has access to that file but root hasn't Affected #: 1 file diff -r 359a822b2e5da89973ce042942a9a10dd2b229e1 -r 3f05316ce541248c18ebc0700e1bb64e77e23e79 scripts/drmaa_external_runner.py --- a/scripts/drmaa_external_runner.py +++ b/scripts/drmaa_external_runner.py @@ -53,6 +53,14 @@ sys.stderr.write("error: User name (%s) is not valid.\n" % username) exit(1) return pw.pw_uid + +def json_file_exists(json_filename): + if not os.path.exists(json_filename): + sys.stderr.write("error: JobTemplate file (%s) doesn't exist\n" % ( json_filename ) ) + exit(1) + + return True + def validate_paramters(): if len(sys.argv)<3: sys.stderr.write("usage: %s [USER-ID] [JSON-JOB-TEMPLATE-FILE]\n" % sys.argv[0]) @@ -70,10 +78,6 @@ sys.stderr.write("error: userid must not be 0 (root)\n") exit(1) - if not os.path.exists(json_filename): - sys.stderr.write("error: JobTemplate file (%s) doesn't exist\n" % ( json_filename ) ) - exit(1) - return uid, json_filename def set_user(uid): @@ -81,8 +85,15 @@ # Get user's default group and set it to current process to make sure file permissions are inherited correctly # Solves issue with permission denied for JSON files gid = pwd.getpwuid(uid).pw_gid + import grp os.setgid(gid) + # Added lines to assure read/write permission for groups + user = pwd.getpwuid(uid).pw_name + groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem] + + os.setgroups(groups) os.setuid(uid) + except OSError, e: if e.errno == errno.EPERM: sys.stderr.write("error: setuid(%d) failed: permission denied. Did you setup 'sudo' correctly for this script?\n" % uid ) @@ -97,7 +108,8 @@ exit(1) def main(): userid, json_filename = validate_paramters() - set_user(userid) + set_user(userid) + json_file_exists(json_filename) s = drmaa.Session() s.initialize() jt = s.createJobTemplate() https://bitbucket.org/galaxy/galaxy-central/commits/9368ca5624c8/ Changeset: 9368ca5624c8 User: jmchilton Date: 2013-12-10 15:29:16 Summary: Merge pull request #267. Affected #: 1 file diff -r 292d68d7e6a101b4a8df06a9f09aa3d16403e14e -r 9368ca5624c8d7a2a84a6007e3a53a317a3e6a91 scripts/drmaa_external_runner.py --- a/scripts/drmaa_external_runner.py +++ b/scripts/drmaa_external_runner.py @@ -53,6 +53,14 @@ sys.stderr.write("error: User name (%s) is not valid.\n" % username) exit(1) return pw.pw_uid + +def json_file_exists(json_filename): + if not os.path.exists(json_filename): + sys.stderr.write("error: JobTemplate file (%s) doesn't exist\n" % ( json_filename ) ) + exit(1) + + return True + def validate_paramters(): if len(sys.argv)<3: sys.stderr.write("usage: %s [USER-ID] [JSON-JOB-TEMPLATE-FILE]\n" % sys.argv[0]) @@ -70,10 +78,6 @@ sys.stderr.write("error: userid must not be 0 (root)\n") exit(1) - if not os.path.exists(json_filename): - sys.stderr.write("error: JobTemplate file (%s) doesn't exist\n" % ( json_filename ) ) - exit(1) - return uid, json_filename def set_user(uid): @@ -81,8 +85,15 @@ # Get user's default group and set it to current process to make sure file permissions are inherited correctly # Solves issue with permission denied for JSON files gid = pwd.getpwuid(uid).pw_gid + import grp os.setgid(gid) + # Added lines to assure read/write permission for groups + user = pwd.getpwuid(uid).pw_name + groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem] + + os.setgroups(groups) os.setuid(uid) + except OSError, e: if e.errno == errno.EPERM: sys.stderr.write("error: setuid(%d) failed: permission denied. Did you setup 'sudo' correctly for this script?\n" % uid ) @@ -97,7 +108,8 @@ exit(1) def main(): userid, json_filename = validate_paramters() - set_user(userid) + set_user(userid) + json_file_exists(json_filename) s = drmaa.Session() s.initialize() jt = s.createJobTemplate() https://bitbucket.org/galaxy/galaxy-central/commits/14487c8ad2ac/ Changeset: 14487c8ad2ac User: jmchilton Date: 2013-12-10 15:02:17 Summary: PEP-8 fixes for scripts/drmaa_external_runner.py. Affected #: 1 file diff -r 9368ca5624c8d7a2a84a6007e3a53a317a3e6a91 -r 14487c8ad2ac94bc51e62b3bb1c8ad5b37c8bc89 scripts/drmaa_external_runner.py --- a/scripts/drmaa_external_runner.py +++ b/scripts/drmaa_external_runner.py @@ -14,7 +14,7 @@ #import simplejson as json #import drmaa new_path = [ os.path.join( os.getcwd(), "lib" ) ] -new_path.extend( sys.path[1:] ) # remove scripts/ from the path +new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy import eggs @@ -25,27 +25,30 @@ import drmaa DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', - 'jobName','email','project' ] + 'jobName', 'email', 'project' ] + def load_job_template_from_file(jt, filename): - f = open(filename,'r') + f = open(filename, 'r') data = json.load(f) for attr in DRMAA_jobTemplate_attributes: if attr in data: setattr(jt, attr, data[attr]) + def valid_numeric_userid(userid): try: uid = int(userid) except: return False try: - pw = pwd.getpwuid(uid) + pwd.getpwuid(uid) except KeyError: sys.stderr.write("error: User-ID (%d) is not valid.\n" % uid) exit(1) return True + def get_user_id_by_name(username): try: pw = pwd.getpwnam(username) @@ -53,16 +56,19 @@ sys.stderr.write("error: User name (%s) is not valid.\n" % username) exit(1) return pw.pw_uid - + + def json_file_exists(json_filename): if not os.path.exists(json_filename): sys.stderr.write("error: JobTemplate file (%s) doesn't exist\n" % ( json_filename ) ) exit(1) - + return True + def validate_paramters(): - if len(sys.argv)<3: + + if len(sys.argv) < 3: sys.stderr.write("usage: %s [USER-ID] [JSON-JOB-TEMPLATE-FILE]\n" % sys.argv[0]) exit(1) @@ -80,6 +86,7 @@ return uid, json_filename + def set_user(uid): try: # Get user's default group and set it to current process to make sure file permissions are inherited correctly @@ -96,16 +103,20 @@ except OSError, e: if e.errno == errno.EPERM: - sys.stderr.write("error: setuid(%d) failed: permission denied. Did you setup 'sudo' correctly for this script?\n" % uid ) + sys.stderr.write( "error: setuid(%d) failed: permission denied. Did you setup 'sudo' correctly for this script?\n" % uid ) exit(1) else: pass - if os.getuid()==0: - sys.stderr.write("error: UID is 0 (root) after changing user. This script should not be run as root. aborting.\n" ) + + if os.getuid() == 0: + sys.stderr.write( "error: UID is 0 (root) after changing user. This script should not be run as root. aborting.\n" ) exit(1) - if os.geteuid()==0: - sys.stderr.write("error: EUID is 0 (root) after changing user. This script should not be run as root. aborting.\n" ) + + if os.geteuid() == 0: + sys.stderr.write( "error: EUID is 0 (root) after changing user. This script should not be run as root. aborting.\n" ) exit(1) + + def main(): userid, json_filename = validate_paramters() set_user(userid) @@ -124,4 +135,3 @@ if __name__ == "__main__": main() - https://bitbucket.org/galaxy/galaxy-central/commits/f7d0ed0413ea/ Changeset: f7d0ed0413ea User: jmchilton Date: 2013-12-10 15:26:22 Summary: Rework contribution from pull request #267 to be optional. To obtain the behavior (assigning all groups for a given user for each job submission) change the property - 'drmaa_external_runjob_script' from 'scripts/drmaa_external_runner.py' to 'scripts/drmaa_external_runner.py --assign_all_groups'. Making this optional because my concern is that this could be an expensive operation on some clusters. Affected #: 2 files diff -r 14487c8ad2ac94bc51e62b3bb1c8ad5b37c8bc89 -r f7d0ed0413ea4f8ff4473b7bc97a49e8fb16b8f5 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -302,7 +302,15 @@ The external script will be run with sudo, and will setuid() to the specified user. Effectively, will QSUB as a different user (then the one used by Galaxy). """ - p = subprocess.Popen([ '/usr/bin/sudo', '-E', self.external_runJob_script, str(username), jobtemplate_filename ], + script_parts = self.external_runJob_script.split() + script = script_parts[0] + command = [ '/usr/bin/sudo', '-E', script] + for script_argument in script_parts[1:]: + command.append(script_argument) + + command.extend( [ str(username), jobtemplate_filename ] ) + log.info("Running command %s" % command) + p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = p.communicate() exitcode = p.returncode diff -r 14487c8ad2ac94bc51e62b3bb1c8ad5b37c8bc89 -r f7d0ed0413ea4f8ff4473b7bc97a49e8fb16b8f5 scripts/drmaa_external_runner.py --- a/scripts/drmaa_external_runner.py +++ b/scripts/drmaa_external_runner.py @@ -67,6 +67,10 @@ def validate_paramters(): + assign_all_groups = False + if "--assign_all_groups" in sys.argv: + assign_all_groups = True + sys.argv.remove("--assign_all_groups") if len(sys.argv) < 3: sys.stderr.write("usage: %s [USER-ID] [JSON-JOB-TEMPLATE-FILE]\n" % sys.argv[0]) @@ -84,21 +88,22 @@ sys.stderr.write("error: userid must not be 0 (root)\n") exit(1) - return uid, json_filename + return uid, json_filename, assign_all_groups -def set_user(uid): +def set_user(uid, assign_all_groups): try: # Get user's default group and set it to current process to make sure file permissions are inherited correctly # Solves issue with permission denied for JSON files gid = pwd.getpwuid(uid).pw_gid import grp os.setgid(gid) - # Added lines to assure read/write permission for groups - user = pwd.getpwuid(uid).pw_name - groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem] + if assign_all_groups: + # Added lines to assure read/write permission for groups + user = pwd.getpwuid(uid).pw_name + groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem] - os.setgroups(groups) + os.setgroups(groups) os.setuid(uid) except OSError, e: @@ -118,8 +123,8 @@ def main(): - userid, json_filename = validate_paramters() - set_user(userid) + userid, json_filename, assign_all_groups = validate_paramters() + set_user(userid, assign_all_groups) json_file_exists(json_filename) s = drmaa.Session() s.initialize() https://bitbucket.org/galaxy/galaxy-central/commits/b8efe5c848f0/ Changeset: b8efe5c848f0 Branch: fretn/fix-for-two-issues-the-secondary-user-1385478183593 User: jmchilton Date: 2013-12-10 15:36:12 Summary: Close branch. Affected #: 0 files Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.