1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
# testClasses.py
# --------------
# Licensing Information: You are free to use or extend these projects for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
#
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
# The core projects and autograders were primarily created by John DeNero
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
# Student side autograding was added by Brad Miller, Nick Hay, and
# Pieter Abbeel (pabbeel@cs.berkeley.edu).
# import modules from python standard library
import inspect
import re
import sys
# Class which models a question in a project. Note that questions have a
# maximum number of points they are worth, and are composed of a series of
# test cases
class Question(object):
def raiseNotDefined(self):
print 'Method not implemented: %s' % inspect.stack()[1][3]
sys.exit(1)
def __init__(self, questionDict, display):
self.maxPoints = int(questionDict['max_points'])
self.testCases = []
self.display = display
def getDisplay(self):
return self.display
def getMaxPoints(self):
return self.maxPoints
# Note that 'thunk' must be a function which accepts a single argument,
# namely a 'grading' object
def addTestCase(self, testCase, thunk):
self.testCases.append((testCase, thunk))
def execute(self, grades):
self.raiseNotDefined()
# Question in which all test cases must be passed in order to receive credit
class PassAllTestsQuestion(Question):
def execute(self, grades):
# TODO: is this the right way to use grades? The autograder doesn't seem to use it.
testsFailed = False
grades.assignZeroCredit()
for _, f in self.testCases:
if not f(grades):
testsFailed = True
if testsFailed:
grades.fail("Tests failed.")
else:
grades.assignFullCredit()
class ExtraCreditPassAllTestsQuestion(Question):
def __init__(self, questionDict, display):
Question.__init__(self, questionDict, display)
self.extraPoints = int(questionDict['extra_points'])
def execute(self, grades):
# TODO: is this the right way to use grades? The autograder doesn't seem to use it.
testsFailed = False
grades.assignZeroCredit()
for _, f in self.testCases:
if not f(grades):
testsFailed = True
if testsFailed:
grades.fail("Tests failed.")
else:
grades.assignFullCredit()
grades.addPoints(self.extraPoints)
# Question in which predict credit is given for test cases with a ``points'' property.
# All other tests are mandatory and must be passed.
class HackedPartialCreditQuestion(Question):
def execute(self, grades):
# TODO: is this the right way to use grades? The autograder doesn't seem to use it.
grades.assignZeroCredit()
points = 0
passed = True
for testCase, f in self.testCases:
testResult = f(grades)
if "points" in testCase.testDict:
if testResult: points += float(testCase.testDict["points"])
else:
passed = passed and testResult
## FIXME: Below terrible hack to match q3's logic
if int(points) == self.maxPoints and not passed:
grades.assignZeroCredit()
else:
grades.addPoints(int(points))
class Q6PartialCreditQuestion(Question):
"""Fails any test which returns False, otherwise doesn't effect the grades object.
Partial credit tests will add the required points."""
def execute(self, grades):
grades.assignZeroCredit()
results = []
for _, f in self.testCases:
results.append(f(grades))
if False in results:
grades.assignZeroCredit()
class PartialCreditQuestion(Question):
"""Fails any test which returns False, otherwise doesn't effect the grades object.
Partial credit tests will add the required points."""
def execute(self, grades):
grades.assignZeroCredit()
for _, f in self.testCases:
if not f(grades):
grades.assignZeroCredit()
grades.fail("Tests failed.")
return False
class NumberPassedQuestion(Question):
"""Grade is the number of test cases passed."""
def execute(self, grades):
grades.addPoints([f(grades) for _, f in self.testCases].count(True))
# Template modeling a generic test case
class TestCase(object):
def raiseNotDefined(self):
print 'Method not implemented: %s' % inspect.stack()[1][3]
sys.exit(1)
def getPath(self):
return self.path
def __init__(self, question, testDict):
self.question = question
self.testDict = testDict
self.path = testDict['path']
self.messages = []
def __str__(self):
self.raiseNotDefined()
def execute(self, grades, moduleDict, solutionDict):
self.raiseNotDefined()
def writeSolution(self, moduleDict, filePath):
self.raiseNotDefined()
return True
# Tests should call the following messages for grading
# to ensure a uniform format for test output.
#
# TODO: this is hairy, but we need to fix grading.py's interface
# to get a nice hierarchical project - question - test structure,
# then these should be moved into Question proper.
def testPass(self, grades):
grades.addMessage('PASS: %s' % (self.path,))
for line in self.messages:
grades.addMessage(' %s' % (line,))
return True
def testFail(self, grades):
grades.addMessage('FAIL: %s' % (self.path,))
for line in self.messages:
grades.addMessage(' %s' % (line,))
return False
# This should really be question level?
#
def testPartial(self, grades, points, maxPoints):
grades.addPoints(points)
extraCredit = max(0, points - maxPoints)
regularCredit = points - extraCredit
grades.addMessage('%s: %s (%s of %s points)' % ("PASS" if points >= maxPoints else "FAIL", self.path, regularCredit, maxPoints))
if extraCredit > 0:
grades.addMessage('EXTRA CREDIT: %s points' % (extraCredit,))
for line in self.messages:
grades.addMessage(' %s' % (line,))
return True
def addMessage(self, message):
self.messages.extend(message.split('\n'))
|
