Commit 6a0b4fe5 authored by Alan Marchiori's avatar Alan Marchiori
Browse files

begin grade

parent d791790c
......@@ -2,4 +2,5 @@ from .init import init
from .check import check
from .userconfig import userconfig
from .test import test
__all__ = ['userconfig', 'init', 'check', 'test']
from .grade import grade
__all__ = ['userconfig', 'init', 'check', 'test', 'grade']
......@@ -14,187 +14,11 @@ from pathlib import Path
import shutil
import difflib
from commands.checker import Checker
def dbg(x):
debug(__name__ + x)
class Checker:
def __init__(self, checkinfo, labroot):
self.info = checkinfo
self.cwd = labroot
def execute(self, args):
"execute a command and optionally examine results"
dbg('.execute({})'.format(args))
results = []
for test in args:
# plain string is just a command (ignore output)
if isinstance(test, str):
c,t = run(test)
else:
# dicts can be used to specify tests to perform
for cmd, check in test.items():
sh = True if 'shell' in check and check['shell'] else False
instr = check['stdin'] if 'stdin' in check else None
to = check['timeout'] if 'timeout' in check else 2
# check for multiline inputs
if instr:
if type(instr) == list:
instr = "\n".join(instr)
if type(instr) != str:
warn("Rubric has non-string stdin ({})!".format(
type(instr)))
instr = str(instr)
if to > 2:
warn("Running {} with timeout {}s, please wait!".format(
cmd, to
))
try:
c,t = run(cmd, shell=sh, input=instr, timeout=to)
except subprocess.TimeoutExpired:
error("{}: Command timeout!".format(cmd))
c = -1
t = ""
msg = check['message'] if 'message' in check else "$ {}".format(cmd)
if 'returncode' in check:
results += [c == check['returncode']]
if results[-1]:
success("{}: success.".format(msg))
else:
error("{}: failed!".format(msg))
error(t)
# stop on first error
return False
if 'stdout' in check:
if type(check['stdout']) == list:
x = "\n".join(check['stdout'])
elif type(check['stdout']) == str:
x = check['stdout']
else:
warn("Rubric has non-string stdout!")
x = str(check['stdout'])
results += [t == x]
if results[-1]:
success("{}: Output passed.".format(msg))
else:
showdiff = True
if 'diff' in check:
showdiff = check['diff']
if showdiff:
error("{}: Output not as expected (check formatting!)".format(msg))
for line in difflib.context_diff(
x.split("\n"), t.split("\n"),
fromfile='Expected output',
tofile='Your output'):
echo(line.strip())
else:
error("{}: failed.".format(msg))
return False
return all(results)
def exists(self, args):
"file existence check, args should be a list"
dbg('.exists({})'.format(args))
for fname, result in zip(args, map(os.path.exists, args)):
if not result:
error("The file '{}' does not exist!".format(fname))
return False
return True
def is_empty(self, args):
"true if all files in args (list) exist and have size 0 bytes"
dbg('.is_empty({})'.format(args))
if self.exists(args):
return all(map(lambda x: os.stat(x).st_size == 0, args))
else:
return False
def wc(self, args):
"word count, args have optional checks"
dbg('.wc({})'.format(args))
result = []
for test in args:
for fname, check in test.items():
c, t = run('wc {}'.format(fname))
#t = newlines, words, bytes, filename
t = t.split()
for chk, value in check.items():
if chk == 'min_lines':
a = int(t[0]) >= int(value)
result += [a]
if not a:
warn("The file {} is too short.".format(
fname
))
else:
raise Exception("Unknown wc check constraint {}".format(chk))
return all(result)
def contains(self, args):
"list of file: [words] to check for"
dbg('.contains({})'.format(args))
result = []
for test in args:
for fname, check in test.items():
text = Path(fname).read_text()
for c in check:
a = c in text
result += [a]
if not a:
warn("The file {} does not contain \"{}\"".format(
fname, c
))
return all(result)
def cd_labroot(self):
if os.path.exists(self.cwd):
debug("{}: {}".format(__name__,
"cd into {}".format(self.cwd)))
os.chdir(self.cwd)
return True
else:
error("The lab should be in the folder {}, but this path doesn't exist!".format(self.cwd))
return False
def do_check(self):
"""Does the actual check defined in the dict/json object
The cwd is set to labroot (and restored on exit)
Returns True if all tests pass
Returns False if any test fails
"""
incwd = os.getcwd()
try:
if self.cd_labroot():
r = []
# for k, args in self.info.items():
# if k in ['index', 'name', 'check',
# 'prompt', 'points',
# 'on_error', 'on_pass', # these are rubric attributes
# 'show']: # show is used for grading
# continue
for subpart in self.info["check"]:
for k, args in subpart.items():
debug(k, args)
if hasattr(self, k):
r += [getattr(self, k)(args)]
if not r[-1]:
#stop on first error
return False
else:
warn("I don't know what {} means.".format(k))
return all(r)
else:
return False
finally:
os.chdir(incwd)
@click.command(short_help="Check LAB")
@click.option('--lab', type=int, default=None)
@click.option('--part', type=int, default=None)
......
"""
This is the class that performs automated tests.
It is use by check and grade!
"""
class Checker:
def __init__(self, checkinfo, labroot):
self.info = checkinfo
self.cwd = labroot
def execute(self, args):
"execute a command and optionally examine results"
dbg('.execute({})'.format(args))
results = []
for test in args:
# plain string is just a command (ignore output)
if isinstance(test, str):
c,t = run(test)
else:
# dicts can be used to specify tests to perform
for cmd, check in test.items():
sh = True if 'shell' in check and check['shell'] else False
instr = check['stdin'] if 'stdin' in check else None
to = check['timeout'] if 'timeout' in check else 2
# check for multiline inputs
if instr:
if type(instr) == list:
instr = "\n".join(instr)
if type(instr) != str:
warn("Rubric has non-string stdin ({})!".format(
type(instr)))
instr = str(instr)
if to > 2:
warn("Running {} with timeout {}s, please wait!".format(
cmd, to
))
try:
c,t = run(cmd, shell=sh, input=instr, timeout=to)
except subprocess.TimeoutExpired:
error("{}: Command timeout!".format(cmd))
c = -1
t = ""
msg = check['message'] if 'message' in check else "$ {}".format(cmd)
if 'returncode' in check:
results += [c == check['returncode']]
if results[-1]:
success("{}: success.".format(msg))
else:
error("{}: failed!".format(msg))
error(t)
# stop on first error
return False
if 'stdout' in check:
if type(check['stdout']) == list:
x = "\n".join(check['stdout'])
elif type(check['stdout']) == str:
x = check['stdout']
else:
warn("Rubric has non-string stdout!")
x = str(check['stdout'])
results += [t == x]
if results[-1]:
success("{}: Output passed.".format(msg))
else:
showdiff = True
if 'diff' in check:
showdiff = check['diff']
if showdiff:
error("{}: Output not as expected (check formatting!)".format(msg))
for line in difflib.context_diff(
x.split("\n"), t.split("\n"),
fromfile='Expected output',
tofile='Your output'):
echo(line.strip())
else:
error("{}: failed.".format(msg))
return False
return all(results)
def exists(self, args):
"file existence check, args should be a list"
dbg('.exists({})'.format(args))
for fname, result in zip(args, map(os.path.exists, args)):
if not result:
error("The file '{}' does not exist!".format(fname))
return False
return True
def is_empty(self, args):
"true if all files in args (list) exist and have size 0 bytes"
dbg('.is_empty({})'.format(args))
if self.exists(args):
return all(map(lambda x: os.stat(x).st_size == 0, args))
else:
return False
def wc(self, args):
"word count, args have optional checks"
dbg('.wc({})'.format(args))
result = []
for test in args:
for fname, check in test.items():
c, t = run('wc {}'.format(fname))
#t = newlines, words, bytes, filename
t = t.split()
for chk, value in check.items():
if chk == 'min_lines':
a = int(t[0]) >= int(value)
result += [a]
if not a:
warn("The file {} is too short.".format(
fname
))
else:
raise Exception("Unknown wc check constraint {}".format(chk))
return all(result)
def contains(self, args):
"list of file: [words] to check for"
dbg('.contains({})'.format(args))
result = []
for test in args:
for fname, check in test.items():
text = Path(fname).read_text()
for c in check:
a = c in text
result += [a]
if not a:
warn("The file {} does not contain \"{}\"".format(
fname, c
))
return all(result)
def cd_labroot(self):
if os.path.exists(self.cwd):
debug("{}: {}".format(__name__,
"cd into {}".format(self.cwd)))
os.chdir(self.cwd)
return True
else:
error("The lab should be in the folder {}, but this path doesn't exist!".format(self.cwd))
return False
def do_check(self):
"""Does the actual check defined in the dict/json object
The cwd is set to labroot (and restored on exit)
Returns True if all tests pass
Returns False if any test fails
"""
incwd = os.getcwd()
try:
if self.cd_labroot():
r = []
# for k, args in self.info.items():
# if k in ['index', 'name', 'check',
# 'prompt', 'points',
# 'on_error', 'on_pass', # these are rubric attributes
# 'show']: # show is used for grading
# continue
for subpart in self.info["check"]:
for k, args in subpart.items():
debug(k, args)
if hasattr(self, k):
r += [getattr(self, k)(args)]
if not r[-1]:
#stop on first error
return False
else:
warn("I don't know what {} means.".format(k))
return all(r)
else:
return False
finally:
os.chdir(incwd)
import click
import config
from config.echo import *
import courses
from config import UserConfig
from utils.shell import run
from utils.git import Git
from pprint import pprint
import os
from pathlib import Path
import shutil
import difflib
@click.command(short_help="Grade LAB (TA only)")
@click.option('--lab', type=int)
@click.option('--part', type=int, default=None)
def grade(lab, part):
"""grade LAB and PART for students from gitlab. LAB is required. If PART
is omitted, ALL parts are graded.
"""
# convert lab number to filename magic.
if lab:
labstr = "lab{:02}".format(lab)
else:
labstr = None
coursename, coursepath, labname = courses.detect_course(lab=labstr)
if not coursename:
error("The current directory is not an initialized course! You must first cd into an initialized course to check!")
return
if not labname:
error("Could not detect LAB. Run this command from your lab folder or specify the LAB on the command line (--lab #)")
return
if part:
echo("Checking {} (part {}) of {}".format(labname, part, coursename))
else:
echo("Checking {} of {}".format(labname, coursename))
rubric = courses.load_rubric(coursename, labname)
if not rubric:
error("{} is not defined for the course {}.)".format(
labname, coursename
))
return
width = min(120, shutil.get_terminal_size().columns)
......@@ -26,11 +26,12 @@ def init_ta(coursename, localpath):
error("ABORT: Cannot initialize since the path {} already exists!".format(
localpath))
return
os.makedirs(localpath, mode=0o700)
with config.UserConfig() as uc:
uc.add_ta(
name=coursename,
path=localpath)
success("Done.")
@click.command(short_help='Initialize a course.')
@click.argument('course', callback=validation.course)
......
......@@ -16,6 +16,10 @@ def test():
echo("Your initialized courses: {}".format(
list(uc['courses'].keys())
))
if 'ta' in uc:
echo("You are a TA for: {}".format(
list(uc['ta'].keys())
))
echo("All available courses: {}".format(
list(courses.all.keys())
))
......
......@@ -19,6 +19,8 @@ def detect_course(lab = None):
found = False
cwd = os.path.realpath(os.getcwd())
# TODO support TA courses!
for coursename, coursepath in UserConfig.getConfig()['courses'].items():
debug("detect_course: testing {}, {}".format(coursename, coursepath))
if is_subpath(cwd, os.path.realpath(os.path.expanduser(coursepath))):
......
......@@ -21,7 +21,7 @@ import config
import commands
__version__ = '1.0.1'
__date__ = '2020-01-08T15:12:16.309992'
__date__ = '2020-01-10T08:54:39.265268'
__user__ = 'cs206'
__host__ = 'linuxremote3.bucknell.edu'
@click.group()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment