Merge branch 'master' of gitlab.bucknell.edu:amm042/labtool

.gitignore

@@ -4,3 +4,135 @@ lt.c
 lt
 __pycache__/
 *.py[cod]
+
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
\ No newline at end of file

LICENSE

+This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by-nc-sa/4.0/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
\ No newline at end of file

commands/check.py

@@ -14,6 +14,8 @@ from pathlib import Path
 import shutil
 import difflib
 
+#from utils.decorators import rate_lim
+from utils.ratelimit import is_rate_limited, log_result
 from commands.checker import Checker
 
 def dbg(x):
@@ -69,6 +71,7 @@ def show_report(hist, partstr, info):
     #         hist['grade']['TOTAL']['who']
     #     ))
 
+
 @click.command(short_help="Check LAB")
 @click.option('--lab', type=int, default=None)
 @click.option('--part', type=int, default=None)
@@ -92,10 +95,10 @@ def check(lab, part, **kwargs):
         error("Could not detect LAB. Run this command from your lab folder or specify the LAB on the command line (--lab #)")
         return
 
-    if part:
-        echo("Checking {} (part {}) of {}".format(labname, part, coursename))
-    else:
+    if part is None:
         echo("Checking {} of {}".format(labname, coursename))
+    else:
+        echo("Checking {} (part {}) of {}".format(labname, part, coursename))
 
     rubric = courses.load_rubric(coursename, labname)
 
@@ -107,7 +110,6 @@ def check(lab, part, **kwargs):
     width = min(120, shutil.get_terminal_size().columns)
 
     if ista:
-        print(kwargs)
         error("Check only works in the student context, you are in a TA course path!")
         error("{} is a TA path for {}!".format(
             coursepath, coursename
@@ -122,50 +124,45 @@ def check(lab, part, **kwargs):
     # if History.exists(location=labpath):
     #     return show_report(rubric, labpath)
     #     return
-
-    # with History(location=labpath, save=False) as hist:
+    
     presults = []
     for partstr, info in rubric['parts'].items():
         if isinstance(part, int) and part != info['index']:
-            continue
-
-        # if 'grade' in hist and partstr in hist['grade']:
-        #     show_report(hist, partstr, info)
-        #     continue
+            continue           
 
         if 'check' in info:
-            # if 'check' in hist and partstr in hist['check']:
-            #     # print("partstr: {}".format(partstr))
-            #     # print("pr: {}".format(presults))
-            #     # print("hi: {}".format(hist['check'][partstr]['result']))
-            #     presults += [hist['check'][partstr]['result']]
-            # else:
-            if True:
-                c = Checker(checkinfo=info,
-                            labroot=labpath,
-                            remotepath=rubric['path'])
-
-                newline()
-                xlen = 47 # length of everything without name
-                echo("Part {:3}: {}".format(
-                    info['index'],
-                    info['name'][:max(10,width-xlen)].ljust(width-xlen)
-                    ))
-                echo(info['prompt'])
-                if c.do_check():
-                    presults += [True]
-                    if 'on_pass' in info:
-                        success(info['on_pass'])
-                    else:
-                        success('This part passed!')
+            if is_rate_limited(cmd="check", lab=labname, part=partstr):
+                error(f"Skipping {labname} - part {part} - {partstr} check, since you are out of checks!")
+                continue
+
+            c = Checker(checkinfo=info,
+                        labroot=labpath,
+                        remotepath=rubric['path'])
+
+            newline()
+            xlen = 47 # length of everything without name
+            echo("Part {:3}: {}".format(
+                info['index'],
+                info['name'][:max(10,width-xlen)].ljust(width-xlen)
+                ))
+            echo(info['prompt'])
+            if c.do_check():
+                
+                presults += [True]
+                if 'on_pass' in info:
+                    success(info['on_pass'])
+                else:
+                    success('This part passed!')
+            else:
+                # log failure for rate limting
+                log_result(cmd="check", lab=labname, part=partstr)
+                presults += [False]
+                if 'on_error' in info:
+                    error(info['on_error'])
                 else:
-                    presults += [False]
-                    if 'on_error' in info:
-                        error(info['on_error'])
-                    else:
-                        error('This part failed!')
-                    # stop checking on first part failure
-                    break
+                    error('This part failed!')
+                # stop checking on first part failure
+                break
 
         # show total grade at the end, if it's graded.
         # if 'grade' in hist and 'TOTAL' in hist['grade']:

commands/db.py

@@ -8,11 +8,30 @@ from utils.db import make_connect_str, encrypt_pass, make_password
 import utils.db
 from bson import json_util
 import sys
+
+from utils.ratelimit import reset_limits
+
 @click.group()
 def db():
     "Debug interface for db"
     pass
 
+@db.command()
+@click.argument('username')
+@click.argument('lab', type=int, default=None)
+@click.argument('cmd', default="check")
+def ulim(username, lab, cmd):
+    "Reset all lab command limits for the command, username, lab"
+
+    # convert lab number to filename magic.
+    if lab:
+        labstr = "lab{:02}".format(lab)
+    else:
+        labstr = None
+    echo (f"Reseting all {cmd} limit for {username} on lab {labstr}.")
+
+    echo (reset_limits(cmd, username, labstr))
+
 @db.command()
 def show():
     "shows the db config data"

commands/init.py

@@ -35,7 +35,7 @@ def init_ta(coursename, localpath):
             gl = GitLab()
 
             while not 'api_token' in uc.cfg or not gl.check_token():
-                api_token = prompt("Got to {} and create a personal access token with the \"api scope\" checked. Copy-paste the value here".format(
+                api_token = prompt("Go to {} and create a personal access token with the \"api scope\" checked. Copy-paste the value here".format(
                     config.gitlab_api_token_url
                 ))
                 uc.add_string(
@@ -49,8 +49,8 @@ def init_ta(coursename, localpath):
 
 @click.command(short_help='Initialize a course.')
 @click.argument('course', callback=validation.course)
-@click.argument('section', callback=validation.section)
 @click.argument('semester', callback=validation.semester)
+@click.argument('section', callback=validation.section)
 @click.option('--prefix', default="~/",
     help="Override the default path prefix (~/)")
 @click.option("--ta", default=False,
@@ -58,7 +58,7 @@ def init_ta(coursename, localpath):
               is_flag = True)
 def init(course, semester, section, prefix, ta):
     """Initializes the lab tool structure on your local system
-    and on gitlab for a given COURSE, SECTION, and SEMESTER. Example: init CSCI206 S20 61.
+    and on gitlab for a given COURSE, SEMESTER, and SECTION. Example: init CSCI206 S20 61.
     """
 
     coursename = config.course_name_fmt.format(

config/__init__.py

@@ -53,3 +53,6 @@ crypto_pkey = b"It's really ten years into the future."
 
 # mongodb pass
 #mongodb_pass = 
+
+# logging server
+labtool_url = "http://localhost:3939"
\ No newline at end of file

courses/__init__.py

@@ -79,13 +79,18 @@ def detect_course(lab = None):
 def load_json(filepath):
     "loads a json document from filepath"
     debug('loading JSON config file from {}'.format(filepath))
-    with open(filepath, 'r') as fp:
-        try:
-            return json.load(fp)
-        except json.decoder.JSONDecodeError as x:
-            error("Failed to load rubric at {}".format(filepath))
-            error(x)
-            return None
+
+    try:
+        with open(filepath, 'r') as fp:
+            try:
+                return json.load(fp)
+            except json.decoder.JSONDecodeError as x:
+                error("Failed to load rubric at {}".format(filepath))
+                error(x)
+                return None
+    except PermissionError:
+        error(f"Labtool needs permission to open the file {filepath}.")
+        exit()
 def check_rubric(filename, r):
     "perform sanity checks on rubric and warn user of problems"
     debug("Checking rubric {}".format(filename))
@@ -180,7 +185,13 @@ def load_courses(searchpath):
             for coursename, coursefile in courses.items():
                 #print('reading', os.path.join(pathname, coursefile))
                 #print('split', os.path.split(coursefile))
-                c[coursename] = load_json(os.path.join(pathname, coursefile))
+
+                try:
+                    c[coursename] = load_json(os.path.join(pathname, coursefile))
+                except FileNotFoundError:
+                    error(f"Failed to load course config from {os.path.join(pathname, coursefile)} which is defined in {tpath}.")
+                    exit()
+
                 coursepath = os.path.join(pathname, os.path.split(coursefile)[0])
                 # create full pathnames from relative paths in coursefile
                 def exppath(relpath, pathname):

plotstats/plot.py

+from functools import total_ordering
+from bson import json_util
+import math
+import matplotlib.pyplot as plt
+from collections import defaultdict
+from pprint import pprint
+from operator import itemgetter
+import scipy.stats
+import numpy 
+import os.path
+from pathlib import Path
+#from matplotlib import lines, markers
+from cycler import cycler
+# http://olsgaard.dk/monochrome-black-white-plots-in-matplotlib.html
+# Create cycler object. Use any styling from above you please
+
+monochrome = (cycler('color', ['k']) * cycler('linestyle', ['-', '--', '-.', ':']) )
+
+# plt.rc('text', usetex=True)
+# plt.rc('font', family='serif')
+# http://phyletica.org/matplotlib-fonts/
+import matplotlib
+matplotlib.rcParams['pdf.fonttype'] = 42
+matplotlib.rcParams['ps.fonttype'] = 42
+
+def read(fname='gradestat.json'):
+    for p in [Path("."), Path("..")]:
+        if os.path.exists(p / fname):
+            with open(p / fname, "r") as f:
+                return json_util.loads(f.read())
+    raise FileNotFoundError()
+def extract(obj, attr, filter_func = lambda x: False):
+    return list(filter(filter_func, [x[attr] for x in obj if x != None]))
+def plot1(data, plotattr = "grade_days", print_plotattr = None, max_y = 80):
+    if print_plotattr == None:
+        print_plotattr = plotattr
+    courses = data.keys()
+
+    cols = 2
+    rows = math.ceil(len(courses)/cols)
+
+    fig, axes = plt.subplots(figsize=(5, 5), ncols = cols, nrows = rows)
+
+    for course, ax in zip(courses, axes.flatten()):
+        info = data[course]
+
+        labs = sorted(info.keys())
+        
+        series = [extract(info[lab].values(), plotattr, lambda x: x != None and x > 0) for lab in labs]
+        #series = [list(filter(lambda x: x != None and x > 0, [y[plotattr] for y in info[lab].values()])) for lab in labs]
+        #series = list(filter(lambda x: x != None and x > 0, by_lab))
+        run = True
+        while run:
+            run = False
+            for i, ser in enumerate(series):
+                #drop empty series
+                if len(ser) == 0:
+                    del labs[i]
+                    del series[i]
+                    run = True
+                    break
+
+        print (labs)
+        print (series)
+
+        ax.boxplot(series)
+        ax.set_xticklabels([int(x[3:]) for x in labs]) # strip "Lab"
+        #ax.tick_params(axis='x', rotation = 90)
+        if max([max(i) for i in series]) > max_y:
+            ax.set_ylim(0, max_y)
+        ax.set_title(course)
+        ax.set_ylabel(print_plotattr)
+        ax.set_xlabel("Lab Number")
+    fig.suptitle(print_plotattr)
+    plt.tight_layout()
+
+def plot2(data, plotattr, print_plotattr):
+    # merge sections from each semester
+    sem = {}
+    labs = set()
+    
+    for k,v in data.items():
+        course,semester,section = k.split("-")
+        
+        series = {}
+        
+        for lab in v.keys():
+            #series[lab] = list(filter(lambda x: x != None and x > 0, v[lab].values()))
+            
+            labs.add(lab)
+            series[lab] = extract(v[lab].values(), plotattr, lambda x: x != None and x > 0)
+                
+        sem[semester] = series
+
+    #labs = sorted(labs)
+    empty = []
+    print("have labs", labs)
+    for s,v in sem.items():
+        empty = [k for k,sub in v.items() if len(sub) == 0]
+        print("removing ",empty)
+        for e in empty:
+            del v[e]
+            if e in labs:
+                labs.remove(e)
+
+    labs = sorted(labs)
+    
+    rows = 2
+    cols = math.ceil(len(sem)/rows)
+
+    fig, axes = plt.subplots(figsize=(8, 8), ncols = cols, nrows = rows)
+
+    #fig, ax = plt.subplots(figsize=(5, 5))
+
+    titles = {'S20': 'Spring 2020',
+            'S21': 'Spring 2021'}
+    for seminfo, ax in zip(sem.items(), axes):  
+        semester, data = seminfo
+                        
+        ax.boxplot([data[lab] for lab in labs])
+        ax.set_xticklabels([int(x[3:]) for x in labs]) # strip "Lab"    
+        #ax.tick_params(axis='x', rotation = 90)
+
+        if max([max(i) for i in data.values()]) > 80:
+            ax.set_ylim(0, 80)
+        ax.set_title(titles[semester])
+        ax.set_ylabel(print_plotattr)
+        ax.set_xlabel("Lab Number")
+        plt.tight_layout()
+
+def plot3(ax, fig, data, plotattr = "grade_days", print_plotattr = None,max_val= None):
+    # merge sections and lab from each semester
+    sem = defaultdict(list)
+    labs = set()
+    
+    for k,v in data.items():
+        course,semester,section = k.split("-")
+        
+        series = []   
+        checks = []
+        checks_pass = []    
+        for lab in v.keys():
+
+            if plotattr == "checks_passed":
+                checks += extract(v[lab].values(), "checks", lambda x: x != None)
+                checks_pass += extract(v[lab].values(), "checks_passed", lambda x: x != None)
+                series += [cp / c for c, cp in zip(checks, checks_pass)]
+            else:
+                series += extract(v[lab].values(), plotattr, lambda x: x != None and x > 0)
+            labs.add(lab)
+                
+        sem[semester] += series
+    
+    titles = {'S20': '2020',
+            'S21': '2021'}
+
+    #pprint(sem)
+
+    #ax.set_prop_cycle(monochrome)
+
+    sems = sorted(sem.keys())
+    #pprint(sem[sems[0]])
+    #ax.boxplot([sem[s] for s in sems])
+
+    # ax.hist([sem[s] for s in sems], bins='auto' , density=True, cumulative=True, 
+    #         histtype='step', label = [titles[s] for s in sems], lw=1.5, alpha=1, ls=['-', '--'])
+    for s,ls in zip(sems, ['-','--']):
+        #bins = max_val*5 if max_val != None else 25
+
+        ax.hist(sem[s], bins='auto' , density=True, cumulative=True, 
+            histtype='step', label = titles[s], lw=1.5, alpha=1, ls=ls)
+
+        #ax.plot((bins[:-1]+bins[1:])/2, n, marker = 'x')
+
+        #h,edges = numpy.histogram(sem[s], bins='auto', density=True)
+        #ax.plot(h, label=titles[s], cumulative=True, histtype='step')
+
+    for s in sems:
+        print (s, plotattr, ' median --> ', numpy.median(sem[s]))
+
+    if max_val != None:
+        ax.set_xlim(0,max_val)
+    #ax.grid()
+    #ax.set_xticklabels([int(x[3:]) for x in labs]) # strip "Lab"    
+    #ax.set_xticklabels(sems)
+    #ax.tick_params(axis='x', rotation = 90)
+
+    #if max([max(i) for i in data.values()]) > 80:
+    #ax.set_ylim(0, max_y)
+    #ax.set_title(titles[semester])
+    #ax.set_ylabel(print_plotattr)
+
+    ax.set_xlabel(print_plotattr)
+
+def plot4(data):
+    # plot GRADE vs % checks passing
+    # merge sections and lab from each semester
+    grade_by_sem = defaultdict(list)
+    check_by_sem = defaultdict(list)
+    sems = set()
+    
+    for k,v in data.items():
+        course,semester,section = k.split("-")
+        
+        grades = []        
+        checks = []
+        checks_pass = []
+        for lab in v.keys():
+            vals = v[lab].values()
+
+            for user, info in v[lab].items():
+                if info == None:
+                    continue
+                if info['checks_passed'] != None:
+                    print(course, semester, user, info['grade'], info['checks_passed'] / info['checks'])
+                    if info['checks_passed'] > info['checks']:
+                        print("BAD --> ", user)
+                        print ("Fail on ", course, semester, section) 
+                        exit()
+                
+            grades += extract(vals, "grade", lambda x: x != None)
+            checks += extract(vals, "checks", lambda x: x != None)
+            checks_pass += extract(vals, "checks_passed", lambda x: x != None)
+
+            print(checks)
+            print(checks_pass)
+            for c, cp in zip(checks, checks_pass):
+                if cp > c:
+                    print ("Fail on ", course, semester, section)
+                    exit()
+
+        sems.add(semester)
+
+        grade_by_sem[semester] += grades
+        check_by_sem[semester] += [cp / c for c, cp in zip(checks, checks_pass)]
+
+    sems = sorted(sems)
+
+    titles = {'S20': '2020',
+            'S21': '2021'}
+