Commit cf251fb2 authored by Alan Marchiori's avatar Alan Marchiori
Browse files

Merge branch 'master' of gitlab.bucknell.edu:amm042/labtool

parents 58d7ca77 9aaf850d
......@@ -4,3 +4,135 @@ lt.c
lt
__pycache__/
*.py[cod]
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
\ No newline at end of file
This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/4.0/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
\ No newline at end of file
......@@ -14,6 +14,8 @@ from pathlib import Path
import shutil
import difflib
#from utils.decorators import rate_lim
from utils.ratelimit import is_rate_limited, log_result
from commands.checker import Checker
def dbg(x):
......@@ -69,6 +71,7 @@ def show_report(hist, partstr, info):
# hist['grade']['TOTAL']['who']
# ))
@click.command(short_help="Check LAB")
@click.option('--lab', type=int, default=None)
@click.option('--part', type=int, default=None)
......@@ -92,10 +95,10 @@ def check(lab, part, **kwargs):
error("Could not detect LAB. Run this command from your lab folder or specify the LAB on the command line (--lab #)")
return
if part:
echo("Checking {} (part {}) of {}".format(labname, part, coursename))
else:
if part is None:
echo("Checking {} of {}".format(labname, coursename))
else:
echo("Checking {} (part {}) of {}".format(labname, part, coursename))
rubric = courses.load_rubric(coursename, labname)
......@@ -107,7 +110,6 @@ def check(lab, part, **kwargs):
width = min(120, shutil.get_terminal_size().columns)
if ista:
print(kwargs)
error("Check only works in the student context, you are in a TA course path!")
error("{} is a TA path for {}!".format(
coursepath, coursename
......@@ -122,50 +124,45 @@ def check(lab, part, **kwargs):
# if History.exists(location=labpath):
# return show_report(rubric, labpath)
# return
# with History(location=labpath, save=False) as hist:
presults = []
for partstr, info in rubric['parts'].items():
if isinstance(part, int) and part != info['index']:
continue
# if 'grade' in hist and partstr in hist['grade']:
# show_report(hist, partstr, info)
# continue
continue
if 'check' in info:
# if 'check' in hist and partstr in hist['check']:
# # print("partstr: {}".format(partstr))
# # print("pr: {}".format(presults))
# # print("hi: {}".format(hist['check'][partstr]['result']))
# presults += [hist['check'][partstr]['result']]
# else:
if True:
c = Checker(checkinfo=info,
labroot=labpath,
remotepath=rubric['path'])
newline()
xlen = 47 # length of everything without name
echo("Part {:3}: {}".format(
info['index'],
info['name'][:max(10,width-xlen)].ljust(width-xlen)
))
echo(info['prompt'])
if c.do_check():
presults += [True]
if 'on_pass' in info:
success(info['on_pass'])
else:
success('This part passed!')
if is_rate_limited(cmd="check", lab=labname, part=partstr):
error(f"Skipping {labname} - part {part} - {partstr} check, since you are out of checks!")
continue
c = Checker(checkinfo=info,
labroot=labpath,
remotepath=rubric['path'])
newline()
xlen = 47 # length of everything without name
echo("Part {:3}: {}".format(
info['index'],
info['name'][:max(10,width-xlen)].ljust(width-xlen)
))
echo(info['prompt'])
if c.do_check():
presults += [True]
if 'on_pass' in info:
success(info['on_pass'])
else:
success('This part passed!')
else:
# log failure for rate limting
log_result(cmd="check", lab=labname, part=partstr)
presults += [False]
if 'on_error' in info:
error(info['on_error'])
else:
presults += [False]
if 'on_error' in info:
error(info['on_error'])
else:
error('This part failed!')
# stop checking on first part failure
break
error('This part failed!')
# stop checking on first part failure
break
# show total grade at the end, if it's graded.
# if 'grade' in hist and 'TOTAL' in hist['grade']:
......
......@@ -8,11 +8,30 @@ from utils.db import make_connect_str, encrypt_pass, make_password
import utils.db
from bson import json_util
import sys
from utils.ratelimit import reset_limits
@click.group()
def db():
"Debug interface for db"
pass
@db.command()
@click.argument('username')
@click.argument('lab', type=int, default=None)
@click.argument('cmd', default="check")
def ulim(username, lab, cmd):
"Reset all lab command limits for the command, username, lab"
# convert lab number to filename magic.
if lab:
labstr = "lab{:02}".format(lab)
else:
labstr = None
echo (f"Reseting all {cmd} limit for {username} on lab {labstr}.")
echo (reset_limits(cmd, username, labstr))
@db.command()
def show():
"shows the db config data"
......
......@@ -35,7 +35,7 @@ def init_ta(coursename, localpath):
gl = GitLab()
while not 'api_token' in uc.cfg or not gl.check_token():
api_token = prompt("Got to {} and create a personal access token with the \"api scope\" checked. Copy-paste the value here".format(
api_token = prompt("Go to {} and create a personal access token with the \"api scope\" checked. Copy-paste the value here".format(
config.gitlab_api_token_url
))
uc.add_string(
......@@ -49,8 +49,8 @@ def init_ta(coursename, localpath):
@click.command(short_help='Initialize a course.')
@click.argument('course', callback=validation.course)
@click.argument('section', callback=validation.section)
@click.argument('semester', callback=validation.semester)
@click.argument('section', callback=validation.section)
@click.option('--prefix', default="~/",
help="Override the default path prefix (~/)")
@click.option("--ta", default=False,
......@@ -58,7 +58,7 @@ def init_ta(coursename, localpath):
is_flag = True)
def init(course, semester, section, prefix, ta):
"""Initializes the lab tool structure on your local system
and on gitlab for a given COURSE, SECTION, and SEMESTER. Example: init CSCI206 S20 61.
and on gitlab for a given COURSE, SEMESTER, and SECTION. Example: init CSCI206 S20 61.
"""
coursename = config.course_name_fmt.format(
......
......@@ -53,3 +53,6 @@ crypto_pkey = b"It's really ten years into the future."
# mongodb pass
#mongodb_pass =
# logging server
labtool_url = "http://localhost:3939"
\ No newline at end of file
......@@ -79,13 +79,18 @@ def detect_course(lab = None):
def load_json(filepath):
"loads a json document from filepath"
debug('loading JSON config file from {}'.format(filepath))
with open(filepath, 'r') as fp:
try:
return json.load(fp)
except json.decoder.JSONDecodeError as x:
error("Failed to load rubric at {}".format(filepath))
error(x)
return None
try:
with open(filepath, 'r') as fp:
try:
return json.load(fp)
except json.decoder.JSONDecodeError as x:
error("Failed to load rubric at {}".format(filepath))
error(x)
return None
except PermissionError:
error(f"Labtool needs permission to open the file {filepath}.")
exit()
def check_rubric(filename, r):
"perform sanity checks on rubric and warn user of problems"
debug("Checking rubric {}".format(filename))
......@@ -180,7 +185,13 @@ def load_courses(searchpath):
for coursename, coursefile in courses.items():
#print('reading', os.path.join(pathname, coursefile))
#print('split', os.path.split(coursefile))
c[coursename] = load_json(os.path.join(pathname, coursefile))
try:
c[coursename] = load_json(os.path.join(pathname, coursefile))
except FileNotFoundError:
error(f"Failed to load course config from {os.path.join(pathname, coursefile)} which is defined in {tpath}.")
exit()
coursepath = os.path.join(pathname, os.path.split(coursefile)[0])
# create full pathnames from relative paths in coursefile
def exppath(relpath, pathname):
......
This diff is collapsed.
from functools import total_ordering
from bson import json_util
import math
import matplotlib.pyplot as plt
from collections import defaultdict
from pprint import pprint
from operator import itemgetter
import scipy.stats
import numpy
import os.path
from pathlib import Path
#from matplotlib import lines, markers
from cycler import cycler
# http://olsgaard.dk/monochrome-black-white-plots-in-matplotlib.html
# Create cycler object. Use any styling from above you please
monochrome = (cycler('color', ['k']) * cycler('linestyle', ['-', '--', '-.', ':']) )
# plt.rc('text', usetex=True)
# plt.rc('font', family='serif')
# http://phyletica.org/matplotlib-fonts/
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
def read(fname='gradestat.json'):
for p in [Path("."), Path("..")]:
if os.path.exists(p / fname):
with open(p / fname, "r") as f:
return json_util.loads(f.read())
raise FileNotFoundError()
def extract(obj, attr, filter_func = lambda x: False):
return list(filter(filter_func, [x[attr] for x in obj if x != None]))
def plot1(data, plotattr = "grade_days", print_plotattr = None, max_y = 80):
if print_plotattr == None:
print_plotattr = plotattr
courses = data.keys()
cols = 2
rows = math.ceil(len(courses)/cols)
fig, axes = plt.subplots(figsize=(5, 5), ncols = cols, nrows = rows)
for course, ax in zip(courses, axes.flatten()):
info = data[course]
labs = sorted(info.keys())
series = [extract(info[lab].values(), plotattr, lambda x: x != None and x > 0) for lab in labs]
#series = [list(filter(lambda x: x != None and x > 0, [y[plotattr] for y in info[lab].values()])) for lab in labs]
#series = list(filter(lambda x: x != None and x > 0, by_lab))
run = True
while run:
run = False
for i, ser in enumerate(series):
#drop empty series
if len(ser) == 0:
del labs[i]
del series[i]
run = True
break
print (labs)
print (series)
ax.boxplot(series)
ax.set_xticklabels([int(x[3:]) for x in labs]) # strip "Lab"
#ax.tick_params(axis='x', rotation = 90)
if max([max(i) for i in series]) > max_y:
ax.set_ylim(0, max_y)
ax.set_title(course)
ax.set_ylabel(print_plotattr)
ax.set_xlabel("Lab Number")
fig.suptitle(print_plotattr)
plt.tight_layout()
def plot2(data, plotattr, print_plotattr):
# merge sections from each semester
sem = {}
labs = set()
for k,v in data.items():
course,semester,section = k.split("-")
series = {}
for lab in v.keys():
#series[lab] = list(filter(lambda x: x != None and x > 0, v[lab].values()))
labs.add(lab)
series[lab] = extract(v[lab].values(), plotattr, lambda x: x != None and x > 0)
sem[semester] = series
#labs = sorted(labs)
empty = []
print("have labs", labs)
for s,v in sem.items():
empty = [k for k,sub in v.items() if len(sub) == 0]
print("removing ",empty)
for e in empty:
del v[e]
if e in labs:
labs.remove(e)
labs = sorted(labs)
rows = 2
cols = math.ceil(len(sem)/rows)
fig, axes = plt.subplots(figsize=(8, 8), ncols = cols, nrows = rows)
#fig, ax = plt.subplots(figsize=(5, 5))
titles = {'S20': 'Spring 2020',
'S21': 'Spring 2021'}
for seminfo, ax in zip(sem.items(), axes):
semester, data = seminfo
ax.boxplot([data[lab] for lab in labs])
ax.set_xticklabels([int(x[3:]) for x in labs]) # strip "Lab"
#ax.tick_params(axis='x', rotation = 90)
if max([max(i) for i in data.values()]) > 80:
ax.set_ylim(0, 80)
ax.set_title(titles[semester])
ax.set_ylabel(print_plotattr)
ax.set_xlabel("Lab Number")
plt.tight_layout()
def plot3(ax, fig, data, plotattr = "grade_days", print_plotattr = None,max_val= None):
# merge sections and lab from each semester
sem = defaultdict(list)
labs = set()
for k,v in data.items():
course,semester,section = k.split("-")
series = []
checks = []
checks_pass = []
for lab in v.keys():
if plotattr == "checks_passed":
checks += extract(v[lab].values(), "checks", lambda x: x != None)
checks_pass += extract(v[lab].values(), "checks_passed", lambda x: x != None)
series += [cp / c for c, cp in zip(checks, checks_pass)]
else:
series += extract(v[lab].values(), plotattr, lambda x: x != None and x > 0)
labs.add(lab)
sem[semester] += series
titles = {'S20': '2020',
'S21': '2021'}
#pprint(sem)
#ax.set_prop_cycle(monochrome)
sems = sorted(sem.keys())
#pprint(sem[sems[0]])
#ax.boxplot([sem[s] for s in sems])
# ax.hist([sem[s] for s in sems], bins='auto' , density=True, cumulative=True,
# histtype='step', label = [titles[s] for s in sems], lw=1.5, alpha=1, ls=['-', '--'])
for s,ls in zip(sems, ['-','--']):
#bins = max_val*5 if max_val != None else 25
ax.hist(sem[s], bins='auto' , density=True, cumulative=True,
histtype='step', label = titles[s], lw=1.5, alpha=1, ls=ls)
#ax.plot((bins[:-1]+bins[1:])/2, n, marker = 'x')
#h,edges = numpy.histogram(sem[s], bins='auto', density=True)
#ax.plot(h, label=titles[s], cumulative=True, histtype='step')
for s in sems:
print (s, plotattr, ' median --> ', numpy.median(sem[s]))
if max_val != None:
ax.set_xlim(0,max_val)
#ax.grid()
#ax.set_xticklabels([int(x[3:]) for x in labs]) # strip "Lab"
#ax.set_xticklabels(sems)
#ax.tick_params(axis='x', rotation = 90)
#if max([max(i) for i in data.values()]) > 80:
#ax.set_ylim(0, max_y)
#ax.set_title(titles[semester])
#ax.set_ylabel(print_plotattr)
ax.set_xlabel(print_plotattr)
def plot4(data):
# plot GRADE vs % checks passing
# merge sections and lab from each semester
grade_by_sem = defaultdict(list)
check_by_sem = defaultdict(list)
sems = set()
for k,v in data.items():
course,semester,section = k.split("-")
grades = []
checks = []
checks_pass = []
for lab in v.keys():
vals = v[lab].values()
for user, info in v[lab].items():
if info == None:
continue
if info['checks_passed'] != None:
print(course, semester, user, info['grade'], info['checks_passed'] / info['checks'])
if info['checks_passed'] > info['checks']:
print("BAD --> ", user)
print ("Fail on ", course, semester, section)
exit()
grades += extract(vals, "grade", lambda x: x != None)
checks += extract(vals, "checks", lambda x: x != None)
checks_pass += extract(vals, "checks_passed", lambda x: x != None)
print(checks)
print(checks_pass)
for c, cp in zip(checks, checks_pass):
if cp > c:
print ("Fail on ", course, semester, section)
exit()
sems.add(semester)
grade_by_sem[semester] += grades
check_by_sem[semester] += [cp / c for c, cp in zip(checks, checks_pass)]
sems = sorted(sems)
titles = {'S20': '2020',
'S21': '2021'}