checker.py 11.1 KB
Newer Older
Alan Marchiori's avatar
Alan Marchiori committed
1
2
3
4
"""
This is the class that performs automated tests.
It is use by check and grade!
"""
Alan Marchiori's avatar
Alan Marchiori committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
import click
import config

import subprocess
from config.echo import *
import courses
from config import UserConfig
from utils.shell import run
from utils.git import Git

from pprint import pprint
import os
from pathlib import Path
import shutil
import difflib

def dbg(x):
    debug(__name__ + x)

Alan Marchiori's avatar
Alan Marchiori committed
24
class Checker:
Alan Marchiori's avatar
Alan Marchiori committed
25
26
    def __init__(self, checkinfo, labroot,
                 remotepath = None, gitpath=None, verbose=False):
Alan Marchiori's avatar
Alan Marchiori committed
27
28
        """Executes checks in labroot.
        verbose is set for grading to show more output (commands, etc).
Alan Marchiori's avatar
Alan Marchiori committed
29
30
31
32
33

        remotepath is the path relative to the root of the repo
        gitpath is the path on the gitlab server like amm042/cscs206-s20-62

        these are used to construct the web ide path.
Alan Marchiori's avatar
Alan Marchiori committed
34
        """
Alan Marchiori's avatar
Alan Marchiori committed
35
36
        self.info = checkinfo
        self.cwd = labroot
Alan Marchiori's avatar
Alan Marchiori committed
37
        self.verbose = verbose
Alan Marchiori's avatar
Alan Marchiori committed
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
        self.remotepath = remotepath
        self.gitpath = gitpath

    def webide(self, args):
        "open the gitlab web ide ideally showing the files in args"
        #https://gitlab.bucknell.edu/-/ide/project/amm042/csci206-s20-62/edit/master/-/Labs/Lab01
        url = config.gitlab_url + "-/ide/project/" + self.gitpath + '/edit/master/-/' + self.remotepath


        # TODO if running locally, launch these urls to the web browser,
        # else print the links
        if isinstance(args, str):
            echo("Open this link to view the file: " + url + '/' + args)
        elif isinstance(args, list):
            for s in args:
                echo("Open this link to view the file: " + url + '/' + s)
        else:
            echo("Open this link to view the project: " + url)
Alan Marchiori's avatar
Alan Marchiori committed
56
57
58
59
60
61
62
63
64
65

    def execute(self, args):
        "execute a command and optionally examine results"
        dbg('.execute({})'.format(args))

        results = []
        for test in args:
            # plain string is just a command (ignore output)
            if isinstance(test, str):
                c,t = run(test)
Alan Marchiori's avatar
Alan Marchiori committed
66
67
68
69
70
                if self.verbose:
                    echo("$ {}".format(test))
                    echo(str(t))
                    if c != 0:
                        warn("Return code: {}".format(c))
Alan Marchiori's avatar
Alan Marchiori committed
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
            else:
                # dicts can be used to specify tests to perform
                for cmd, check in test.items():

                    sh = True if 'shell' in check and check['shell'] else False
                    instr = check['stdin'] if 'stdin' in check else None
                    to = check['timeout'] if 'timeout' in check else 2

                    # check for multiline inputs
                    if instr:
                        if type(instr) == list:
                            instr = "\n".join(instr)
                        if type(instr) != str:
                            warn("Rubric has non-string stdin ({})!".format(
                                type(instr)))
                            instr = str(instr)

                    if to > 2:
                        warn("Running {} with timeout {}s, please wait!".format(
                            cmd, to
                        ))
                    try:
                        c,t = run(cmd, shell=sh, input=instr, timeout=to)
                    except subprocess.TimeoutExpired:
                        error("{}: Command timeout!".format(cmd))
                        c = -1
                        t = ""

                    msg = check['message'] if 'message' in check else "$ {}".format(cmd)

                    if 'returncode' in check:
                        results += [c == check['returncode']]
                        if results[-1]:
                            success("{}: success.".format(msg))
                        else:
                            error("{}: failed!".format(msg))
                            error(t)
Alan Marchiori's avatar
Alan Marchiori committed
108
109
110
111

                            if 'on_error' in check:
                                error(check['on_error'])

Alan Marchiori's avatar
Alan Marchiori committed
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
                            # stop on first error
                            return False
                    if 'stdout' in check:
                        if type(check['stdout']) == list:
                            x = "\n".join(check['stdout'])
                        elif type(check['stdout']) == str:
                            x = check['stdout']
                        else:
                            warn("Rubric has non-string stdout!")
                            x = str(check['stdout'])
                        results += [t == x]
                        if results[-1]:
                            success("{}: Output passed.".format(msg))
                        else:
                            showdiff = True
                            if 'diff' in check:
                                showdiff = check['diff']

                            if showdiff:
                                error("{}: Output not as expected (check formatting!)".format(msg))
                                for line in difflib.context_diff(
                                    x.split("\n"), t.split("\n"),
                                    fromfile='Expected output',
                                    tofile='Your output'):
                                    echo(line.strip())
                            else:
                                error("{}: failed.".format(msg))
                            return False

        return all(results)
    def exists(self, args):
        "file existence check, args should be a list"
        dbg('.exists({})'.format(args))
        for fname, result in zip(args, map(os.path.exists, args)):
            if not result:
                error("The file '{}' does not exist!".format(fname))
                return False
        return True

    def is_empty(self, args):
        "true if all files in args (list) exist and have size 0 bytes"
        dbg('.is_empty({})'.format(args))
        if self.exists(args):
            return all(map(lambda x: os.stat(x).st_size == 0, args))
        else:
            return False
Alan Marchiori's avatar
Alan Marchiori committed
158
159
160
161
162
163
164
165
166
    def edit(self, args):
        "open default editor for file or list of files"
        dbg('.edit({})'.format(args))
        if isinstance(args, list):
            args = " ".join(args)
        if confirm("Would you like to view the file(s) {}?".format(
            args
        ), default=True):
            click.edit(filename=args)
Alan Marchiori's avatar
Alan Marchiori committed
167
168
169
170
171
172
    def wc(self, args):
        "word count, args have optional checks"
        dbg('.wc({})'.format(args))
        result = []
        for test in args:
            for fname, check in test.items():
Alan Marchiori's avatar
Alan Marchiori committed
173
174
175
176
177
178
179

                if not os.path.exists(fname):
                    warn("The file {} does not exist, so it could not be checked!".format(
                        fname
                    ))
                    return False

Alan Marchiori's avatar
Alan Marchiori committed
180
181
182
                c, t = run('wc {}'.format(fname))
                #t = newlines, words, bytes, filename
                t = t.split()
Alan Marchiori's avatar
Alan Marchiori committed
183
184
185
186
187
                if len(t)!=4:
                    warn("The file {} could not be checked.".format(
                        fname
                    ))
                    return False
Alan Marchiori's avatar
Alan Marchiori committed
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
                for chk, value in check.items():
                    if chk == 'min_lines':
                        a = int(t[0]) >= int(value)
                        result += [a]
                        if not a:
                            warn("The file {} is too short.".format(
                                fname
                            ))
                    else:
                        raise Exception("Unknown wc check constraint {}".format(chk))
        return all(result)
    def contains(self, args):
        "list of file: [words] to check for"
        dbg('.contains({})'.format(args))
        result = []
        for test in args:
            for fname, check in test.items():
Alan Marchiori's avatar
Alan Marchiori committed
205
206
207
208
209
210
211
212
213
214
215
216
                if os.path.exists(fname):
                    text = Path(fname).read_text()
                    for c in check:
                        a = c in text
                        result += [a]
                        if not a:
                            warn("The file {} does not contain \"{}\"".format(
                                fname, c
                            ))
                else:
                    warn("The file {} does not exist!".format(fname))
                    result += [False]
Alan Marchiori's avatar
Alan Marchiori committed
217
218
        return all(result)

Alan Marchiori's avatar
Alan Marchiori committed
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
    def does_not_contain(self, args):
        "list of file: words to check for... returns true if all words are not found"
        dbg('.does_not_contain({})'.format(args))
        result = []
        for test in args:
            for fname, check in test.items():
                if os.path.exists(fname):
                    text = Path(fname).read_text()
                    for c in check:
                        a = c in text
                        result += [a]
                        if a:
                            warn("The file {} contains \"{}\"".format(
                                fname, c
                            ))
                else:
                    warn("The file {} does not exist!".format(fname))
                    result += [False]
        return not any(result)


Alan Marchiori's avatar
Alan Marchiori committed
240
241
242
243
244
245
246
247
248
249
    def cd_labroot(self):
        if os.path.exists(self.cwd):
            debug("{}: {}".format(__name__,
                "cd into {}".format(self.cwd)))
            os.chdir(self.cwd)
            return True
        else:
            error("The lab should be in the folder {}, but this path doesn't exist!".format(self.cwd))
            return False

Alan Marchiori's avatar
Alan Marchiori committed
250
    def do_check(self, stop_on_error=True, check_type='check'):
Alan Marchiori's avatar
Alan Marchiori committed
251
        """Does the actual check defined in the dict/json object
Alan Marchiori's avatar
Alan Marchiori committed
252
253
254
255
256
257

        if stop_on_error is set, it will stop checking at the first error.

        check_type can be 'check' or 'grade' to access different
        parts of the rubric.

Alan Marchiori's avatar
Alan Marchiori committed
258
        The cwd is set to labroot (and restored on exit)
Alan Marchiori's avatar
Alan Marchiori committed
259

Alan Marchiori's avatar
Alan Marchiori committed
260
261
262
263
        Returns True if all tests pass
        Returns False if any test fails
        """
        incwd = os.getcwd()
Alan Marchiori's avatar
Alan Marchiori committed
264
265
266
        if check_type not in self.info:
            warn("{} not defined in rubric!".format(check_type))
            return True
Alan Marchiori's avatar
Alan Marchiori committed
267
268
269
270
271
272
273
274
275
        try:
            if self.cd_labroot():
                r = []
                # for k, args in self.info.items():
                #     if k in ['index', 'name', 'check',
                #         'prompt', 'points',
                #         'on_error', 'on_pass', # these are rubric attributes
                #         'show']: # show is used for grading
                #         continue
Alan Marchiori's avatar
Alan Marchiori committed
276
                for i,subpart in enumerate(self.info[check_type]):
Alan Marchiori's avatar
Alan Marchiori committed
277

Alan Marchiori's avatar
Alan Marchiori committed
278
279
280
                    for k, args in subpart.items():
                        debug(k, args)
                        if hasattr(self, k):
Alan Marchiori's avatar
Alan Marchiori committed
281
282
283
284

                            if self.verbose:
                                echo("Running check: {}: {}".format(k, args))

Alan Marchiori's avatar
Alan Marchiori committed
285
                            r += [getattr(self, k)(args)]
Alan Marchiori's avatar
Alan Marchiori committed
286
                            if stop_on_error and not r[-1]:
Alan Marchiori's avatar
Alan Marchiori committed
287
288
289
290
291
292
293
294
295
296
297
                                #stop on first error
                                return False
                        else:
                            warn("I don't know what {} means.".format(k))

                return all(r)
            else:
                return False

        finally:
            os.chdir(incwd)