Commit 1ebd3a80 authored by Alan Marchiori's avatar Alan Marchiori
Browse files

impoved grading and checking

parent bd42c2cc
...@@ -20,40 +20,45 @@ from utils.history import History ...@@ -20,40 +20,45 @@ from utils.history import History
def dbg(x): def dbg(x):
debug(__name__ + x) debug(__name__ + x)
def show_report(rubric, labpath): def show_report(hist, partstr, info):
"Load the history file and show a nicely formatted grade report." "show a nicely formatted grade report."
width = min(120, shutil.get_terminal_size().columns) width = min(120, shutil.get_terminal_size().columns)
with History(location=labpath) as hist:
for partstr, info in rubric['parts'].items(): #if 'grade' in info:
if 'grade' in info: newline()
newline() xlen = 47 # length of everything without name
xlen = 47 # length of everything without name echo("Part {:3}: {}".format(
echo("Part {:3}: {}".format( info['index'],
info['index'], info['name'][:max(10,width-xlen)].ljust(width-xlen)
info['name'][:max(10,width-xlen)].ljust(width-xlen) ))
)) echo(info['prompt'])
echo(info['prompt'])
if partstr in hist['grade']:
echo('\tGrade: {} of {}'.format( echo('\tGrade: {} of {}'.format(
hist['grade'][partstr]['grade'], hist['grade'][partstr]['grade'],
hist['grade'][partstr]['total'])) hist['grade'][partstr]['total']))
if hist['grade'][partstr]['comment'] != '': if hist['grade'][partstr]['comment'] != '':
echo('\tComment: {}'.format( echo('\tComment: {}'.format(
hist['grade'][partstr]['comment'])) hist['grade'][partstr]['comment']))
if 'TOTAL' in hist['grade']: else:
newline() warn("Part {:3} is ungraded ({})!".format(
echo("-"*width) info['index'], partstr
newline()
echo("Total: {} of {}".format(
hist['grade']['TOTAL']['grade'],
hist['grade']['TOTAL']['total']
))
echo("Graded by {}".format(
hist['grade']['TOTAL']['who']
)) ))
if 'TOTAL' in hist['grade']:
newline()
echo("-"*width)
newline()
echo("Total: {} of {}".format(
hist['grade']['TOTAL']['grade'],
hist['grade']['TOTAL']['total']
))
echo("Graded by {}".format(
hist['grade']['TOTAL']['who']
))
@click.command(short_help="Check LAB") @click.command(short_help="Check LAB")
@click.option('--lab', type=int, default=None) @click.option('--lab', type=int, default=None)
@click.option('--part', type=int, default=None) @click.option('--part', type=int, default=None)
...@@ -103,38 +108,47 @@ def check(lab, part): ...@@ -103,38 +108,47 @@ def check(lab, part):
labpath = os.path.join(coursepath, rubric['path']) labpath = os.path.join(coursepath, rubric['path'])
# check for a history file to see if the lab has been graded # check for a history file to see if the lab has been graded
if History.exists(location=labpath): # if History.exists(location=labpath):
return show_report(rubric, labpath) # return show_report(rubric, labpath)
return # return
presults = [] with History(location=labpath, save=False) as hist:
for partstr, info in rubric['parts'].items(): presults = []
if 'check' in info: for partstr, info in rubric['parts'].items():
if part and part != info['index']: if isinstance(part, int) and part != info['index']:
continue continue
c = Checker(info, labpath)
if 'grade' in hist and partstr in hist['grade']:
newline() show_report(hist, partstr, info)
xlen = 47 # length of everything without name continue
echo("Part {:3}: {}".format(
info['index'], if 'check' in info:
info['name'][:max(10,width-xlen)].ljust(width-xlen) if 'check' in hist and partstr in hist['check']:
)) presults += hist['check'][partstr]['result']
echo(info['prompt'])
if c.do_check():
presults += [True]
if 'on_pass' in info:
success(info['on_pass'])
else:
success('This part passed!')
else:
presults += [False]
if 'on_error' in info:
error(info['on_error'])
else: else:
error('This part failed!') c = Checker(info, labpath)
# stop checking on first part failure
break newline()
xlen = 47 # length of everything without name
echo("Part {:3}: {}".format(
info['index'],
info['name'][:max(10,width-xlen)].ljust(width-xlen)
))
echo(info['prompt'])
if c.do_check():
presults += [True]
if 'on_pass' in info:
success(info['on_pass'])
else:
success('This part passed!')
else:
presults += [False]
if 'on_error' in info:
error(info['on_error'])
else:
error('This part failed!')
# stop checking on first part failure
break
# only check gitlab if all tests pass # only check gitlab if all tests pass
if all(presults): if all(presults):
......
...@@ -22,13 +22,37 @@ def dbg(x): ...@@ -22,13 +22,37 @@ def dbg(x):
debug(__name__ + x) debug(__name__ + x)
class Checker: class Checker:
def __init__(self, checkinfo, labroot, verbose=False): def __init__(self, checkinfo, labroot,
remotepath = None, gitpath=None, verbose=False):
"""Executes checks in labroot. """Executes checks in labroot.
verbose is set for grading to show more output (commands, etc). verbose is set for grading to show more output (commands, etc).
remotepath is the path relative to the root of the repo
gitpath is the path on the gitlab server like amm042/cscs206-s20-62
these are used to construct the web ide path.
""" """
self.info = checkinfo self.info = checkinfo
self.cwd = labroot self.cwd = labroot
self.verbose = verbose self.verbose = verbose
self.remotepath = remotepath
self.gitpath = gitpath
def webide(self, args):
"open the gitlab web ide ideally showing the files in args"
#https://gitlab.bucknell.edu/-/ide/project/amm042/csci206-s20-62/edit/master/-/Labs/Lab01
url = config.gitlab_url + "-/ide/project/" + self.gitpath + '/edit/master/-/' + self.remotepath
# TODO if running locally, launch these urls to the web browser,
# else print the links
if isinstance(args, str):
echo("Open this link to view the file: " + url + '/' + args)
elif isinstance(args, list):
for s in args:
echo("Open this link to view the file: " + url + '/' + s)
else:
echo("Open this link to view the project: " + url)
def execute(self, args): def execute(self, args):
"execute a command and optionally examine results" "execute a command and optionally examine results"
...@@ -221,7 +245,7 @@ class Checker: ...@@ -221,7 +245,7 @@ class Checker:
# 'show']: # show is used for grading # 'show']: # show is used for grading
# continue # continue
for i,subpart in enumerate(self.info[check_type]): for i,subpart in enumerate(self.info[check_type]):
for k, args in subpart.items(): for k, args in subpart.items():
debug(k, args) debug(k, args)
if hasattr(self, k): if hasattr(self, k):
......
...@@ -116,6 +116,15 @@ def grade(lab, part, clone, dograde, regrade, user, feedback): ...@@ -116,6 +116,15 @@ def grade(lab, part, clone, dograde, regrade, user, feedback):
labpath = os.path.join(coursepath, student, rubric['path']) labpath = os.path.join(coursepath, student, rubric['path'])
if not os.path.isdir(labpath):
error(labpath+ " does not exist!")
if confirm("Do you want to create the student lab directory (this is required to grade)?", default=True):
os.makedirs(labpath)
if not os.path.isdir(labpath):
error("Path doesn't exist, cannot grade!")
continue
# check for grade history # check for grade history
hfile = None hfile = None
with History(labpath) as hist: with History(labpath) as hist:
...@@ -124,11 +133,13 @@ def grade(lab, part, clone, dograde, regrade, user, feedback): ...@@ -124,11 +133,13 @@ def grade(lab, part, clone, dograde, regrade, user, feedback):
hist['check'] = {} hist['check'] = {}
if not 'grade' in hist: if not 'grade' in hist:
hist['grade'] = {} hist['grade'] = {}
for partstr, info in rubric['parts'].items(): for partstr, info in rubric['parts'].items():
if 'check' in info: # part can be zero.
if part and part != info['index']: if isinstance(part, int) and part != info['index']:
continue continue
if 'check' in info:
if not regrade and 'check' in hist and partstr in hist['check']: if not regrade and 'check' in hist and partstr in hist['check']:
echo("Skip part {:3}, already checked (set regrade to re-run).".format(info['index'])) echo("Skip part {:3}, already checked (set regrade to re-run).".format(info['index']))
echo("\tpass? {}".format( echo("\tpass? {}".format(
...@@ -138,6 +149,8 @@ def grade(lab, part, clone, dograde, regrade, user, feedback): ...@@ -138,6 +149,8 @@ def grade(lab, part, clone, dograde, regrade, user, feedback):
c = Checker(info, c = Checker(info,
labpath, labpath,
remotepath=rubric['path'],
gitpath = p['path_with_namespace'],
verbose=True) verbose=True)
# grab output # grab output
...@@ -169,19 +182,15 @@ def grade(lab, part, clone, dograde, regrade, user, feedback): ...@@ -169,19 +182,15 @@ def grade(lab, part, clone, dograde, regrade, user, feedback):
'output': end_capture_output(), 'output': end_capture_output(),
'who': who, 'who': who,
'at': utils.timestamp().isoformat()} 'at': utils.timestamp().isoformat()}
if not dograde: # if not dograde:
# if they don't want to grade, bail out here. # # if they don't want to grade, bail out here.
continue # continue
#
newline() # newline()
echo("-"*width) # echo("-"*width)
newline() # newline()
for partstr, info in rubric['parts'].items(): # for partstr, info in rubric['parts'].items():
if 'grade' in info: if dograde and 'grade' in info:
if part and part != info['index']:
continue
if not regrade and 'grade' in hist and partstr in hist['grade'] and 'grade' in hist['grade'][partstr] and 'comment' in hist['grade'][partstr]: if not regrade and 'grade' in hist and partstr in hist['grade'] and 'grade' in hist['grade'][partstr] and 'comment' in hist['grade'][partstr]:
echo("Skip part {:3}, already graded (set regrade to re-run).".format(info['index'])) echo("Skip part {:3}, already graded (set regrade to re-run).".format(info['index']))
echo("Grade {} of {}: {}".format( echo("Grade {} of {}: {}".format(
...@@ -192,6 +201,9 @@ def grade(lab, part, clone, dograde, regrade, user, feedback): ...@@ -192,6 +201,9 @@ def grade(lab, part, clone, dograde, regrade, user, feedback):
continue continue
tot_pts = info['points'] tot_pts = info['points']
# base defaults for grade and grade_msg
grade = tot_pts
grade_msg = "Good!"
# set default grade to 100% or 0% if test passed/failed. # set default grade to 100% or 0% if test passed/failed.
if partstr in hist['check']: if partstr in hist['check']:
...@@ -208,9 +220,11 @@ def grade(lab, part, clone, dograde, regrade, user, feedback): ...@@ -208,9 +220,11 @@ def grade(lab, part, clone, dograde, regrade, user, feedback):
grade = hist['grade'][partstr]['grade'] grade = hist['grade'][partstr]['grade']
if 'comment' in hist['grade'][partstr]: if 'comment' in hist['grade'][partstr]:
grade_msg = hist['grade'][partstr]['comment'] grade_msg = hist['grade'][partstr]['comment']
c = Checker(info, c = Checker(info,
labpath, labpath,
remotepath=rubric['path'],
gitpath = p['path_with_namespace'],
verbose=True) verbose=True)
xlen = 47 # length of everything without name xlen = 47 # length of everything without name
...@@ -245,8 +259,8 @@ def grade(lab, part, clone, dograde, regrade, user, feedback): ...@@ -245,8 +259,8 @@ def grade(lab, part, clone, dograde, regrade, user, feedback):
else: else:
hist['grade']['TOTAL'] = { hist['grade']['TOTAL'] = {
'grade': sum( 'grade': sum(
[hist['grade'][pt]['grade'] for pt in rubric['parts'].keys()]), [hist['grade'][pt]['grade'] for pt in rubric['parts'].keys() if pt in hist['grade']]),
'total': sum([hist['grade'][pt]['total'] for pt in rubric['parts'].keys()]), 'total': sum([hist['grade'][pt]['total'] for pt in rubric['parts'].keys() if pt in hist['grade']]),
'who': who, 'who': who,
'at': utils.timestamp().isoformat() 'at': utils.timestamp().isoformat()
} }
......
...@@ -31,6 +31,20 @@ def init_ta(coursename, localpath): ...@@ -31,6 +31,20 @@ def init_ta(coursename, localpath):
uc.add_ta( uc.add_ta(
name=coursename, name=coursename,
path=localpath) path=localpath)
gl = GitLab()
while not 'api_token' in uc.cfg or not gl.check_token():
api_token = prompt("Got to {} and create a personal access token with the \"api scope\" checked. Copy-paste the value here".format(
config.gitlab_api_token_url
))
uc.add_string(
name='api_token',
value=api_token
)
# needed to refresh the new token cached in the GitLab module
gl = GitLab()
success("Done.") success("Done.")
@click.command(short_help='Initialize a course.') @click.command(short_help='Initialize a course.')
...@@ -205,7 +219,7 @@ def init(course, semester, section, prefix, ta): ...@@ -205,7 +219,7 @@ def init(course, semester, section, prefix, ta):
else: else:
warn("Failed to initialize gitignore file.") warn("Failed to initialize gitignore file.")
warn(coursedef['.gitignore']) warn(coursedef['.gitignore'])
success("Initialization complete!") success("Initialization complete!")
else: else:
error("Sorry, I was unable to clone the repo (did something go wrong earlier?)!") error("Sorry, I was unable to clone the repo (did something go wrong earlier?)!")
...@@ -45,8 +45,25 @@ class Git: ...@@ -45,8 +45,25 @@ class Git:
return c == 0 return c == 0
def file_diff(self, cwd, files): def file_diff(self, cwd, files):
"return true if we need to push a file" "return true if we need to push a file"
cmd = 'git ls-files -o'
debug("[{}]$ {}".format(cwd, cmd))
c, untracked = shell.run(cmd, cwd=cwd)
debug("\t[{}]: {}".format(c, untracked))
untracked = untracked.split("\n")
if isinstance(files, list): if isinstance(files, list):
for f in files:
if f in untracked:
return True
files = " ".join(files) files = " ".join(files)
else:
if files in untracked:
return True
# this works if the remote file exists, it will return if there are changes
# if the file does not exist on remote, there is no diff!
cmd = "git diff --shortstat {}".format(files) cmd = "git diff --shortstat {}".format(files)
debug("[{}]$ {}".format(cwd, cmd)) debug("[{}]$ {}".format(cwd, cmd))
c, r = shell.run(cmd, cwd=cwd) c, r = shell.run(cmd, cwd=cwd)
......
...@@ -19,11 +19,16 @@ class History(dict): ...@@ -19,11 +19,16 @@ class History(dict):
"class method to determine if a history file exists (w/o creation)" "class method to determine if a history file exists (w/o creation)"
return os.path.exists(os.path.join(location, filename)) return os.path.exists(os.path.join(location, filename))
def __init__(self, location=".", filename='.history.json', *args, **kwargs): def __init__(self, location=".",
filename = '.history.json',
save = True,
*args, **kwargs):
"if save is false, never save the history file (read only copy)!"
dict.__init__(self, *args, **kwargs) dict.__init__(self, *args, **kwargs)
self.filename = filename self.filename = filename
self.pathfile = os.path.join(location, filename) self.pathfile = os.path.join(location, filename)
self.rawjson = None self.rawjson = None
self.save = save
debug("using history file {}".format( debug("using history file {}".format(
self.pathfile self.pathfile
)) ))
...@@ -43,6 +48,8 @@ class History(dict): ...@@ -43,6 +48,8 @@ class History(dict):
def __exit__(self, type, value, traceback): def __exit__(self, type, value, traceback):
"save history" "save history"
debug("HISTORY EXIT") debug("HISTORY EXIT")
if not self.save:
return
# only save history if the target path exists # only save history if the target path exists
if os.path.exists(os.path.dirname(self.pathfile)): if os.path.exists(os.path.dirname(self.pathfile)):
s = json.dumps( s = json.dumps(
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment