Skip to content

Commit 1bd6222

Browse files
fusharveluca93
fushar
authored and
veluca93
committed
Start subtask numbering from 0 instead of 1
1 parent 83685ee commit 1bd6222

6 files changed

Lines changed: 160 additions & 52 deletions

File tree

‎cms/grading/scoretypes/abc.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ def max_scores(self):
394394
score += parameter[0]
395395
if all(self.public_testcases[tc_idx] for tc_idx in target):
396396
public_score += parameter[0]
397-
headers += ["Subtask %d (%g)" % (st_idx + 1, parameter[0])]
397+
headers += ["Subtask %d (%g)" % (st_idx, parameter[0])]
398398

399399
return score, public_score, headers
400400

@@ -464,7 +464,7 @@ def compute_score(self, submission_result):
464464

465465
score += st_score
466466
subtasks.append({
467-
"idx": st_idx + 1,
467+
"idx": st_idx,
468468
# We store the fraction so that an "example" testcase
469469
# with a max score of zero is still properly rendered as
470470
# correct or incorrect.
@@ -475,7 +475,7 @@ def compute_score(self, submission_result):
475475
public_score += st_score
476476
public_subtasks.append(subtasks[-1])
477477
else:
478-
public_subtasks.append({"idx": st_idx + 1,
478+
public_subtasks.append({"idx": st_idx,
479479
"testcases": public_testcases})
480480
ranking_details.append("%g" % round(st_score, 2))
481481

‎cmstestsuite/unit_tests/grading/scoretypes/GroupMinTest.py‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class TestGroupMin(ScoreTypeTestMixin, unittest.TestCase):
3131
def setUp(self):
3232
super().setUp()
3333
self._public_testcases = {
34+
"0_0": True,
3435
"1_0": True,
3536
"1_1": True,
3637
"2_0": True,
@@ -86,8 +87,9 @@ def test_parameter_invalid_testcases_regex_no_match_type(self):
8687
def test_max_scores_regexp(self):
8788
"""Test max score is correct when groups are regexp-defined."""
8889
s1, s2, s3 = 10.5, 30.5, 59
89-
parameters = [[s1, "1_*"], [s2, "2_*"], [s3, "3_*"]]
90-
header = ["Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
90+
parameters = [[0, "0_*"], [s1, "1_*"], [s2, "2_*"], [s3, "3_*"]]
91+
header = ["Subtask 0 (0)",
92+
"Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
9193

9294
# Only group 1_* is public.
9395
public_testcases = dict(self._public_testcases)
@@ -109,8 +111,9 @@ def test_max_scores_regexp(self):
109111
def test_max_scores_number(self):
110112
"""Test max score is correct when groups are number-defined."""
111113
s1, s2, s3 = 10.5, 30.5, 59
112-
parameters = [[s1, 2], [s2, 2], [s3, 2]]
113-
header = ["Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
114+
parameters = [[0, 1], [s1, 2], [s2, 2], [s3, 2]]
115+
header = ["Subtask 0 (0)",
116+
"Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
114117

115118
# Only group 1_* is public.
116119
public_testcases = dict(self._public_testcases)
@@ -131,31 +134,55 @@ def test_max_scores_number(self):
131134

132135
def test_compute_score(self):
133136
s1, s2, s3 = 10.5, 30.5, 59
134-
parameters = [[s1, "1_*"], [s2, "2_*"], [s3, "3_*"]]
137+
parameters = [[0, "0_*"], [s1, "1_*"], [s2, "2_*"], [s3, "3_*"]]
135138
gmin = GroupMin(parameters, self._public_testcases)
136139
sr = self.get_submission_result(self._public_testcases)
137140

138141
# All correct.
139-
self.assertComputeScore(gmin.compute_score(sr),
140-
s1 + s2 + s3, s1, [s1, s2, s3])
142+
self.assertComputeScore(
143+
gmin.compute_score(sr),
144+
s1 + s2 + s3, s1, [0, s1, s2, s3], [
145+
{"idx": 0},
146+
{"idx": 1},
147+
{"idx": 2},
148+
{"idx": 3}
149+
])
141150

142151
# Some non-public subtask is incorrect.
143152
self.set_outcome(sr, "3_1", 0.0)
144-
self.assertComputeScore(gmin.compute_score(sr),
145-
s1 + s2, s1, [s1, s2, 0])
153+
self.assertComputeScore(
154+
gmin.compute_score(sr),
155+
s1 + s2, s1, [0, s1, s2, 0], [
156+
{"idx": 0},
157+
{"idx": 1},
158+
{"idx": 2},
159+
{"idx": 3}
160+
])
146161

147162
# Also the public subtask is incorrect.
148163
self.set_outcome(sr, "1_0", 0.0)
149164
self.set_outcome(sr, "1_1", 0.0)
150165
sr.evaluations[1].outcome = 0.0
151-
self.assertComputeScore(gmin.compute_score(sr),
152-
s2, 0.0, [0, s2, 0])
166+
self.assertComputeScore(
167+
gmin.compute_score(sr),
168+
s2, 0.0, [0, 0, s2, 0], [
169+
{"idx": 0},
170+
{"idx": 1},
171+
{"idx": 2},
172+
{"idx": 3}
173+
])
153174

154175
# Some partial results.
155176
self.set_outcome(sr, "3_0", 0.5)
156177
self.set_outcome(sr, "3_1", 0.1)
157-
self.assertComputeScore(gmin.compute_score(sr),
158-
s2 + s3 * 0.1, 0.0, [0, s2, s3 * 0.1])
178+
self.assertComputeScore(
179+
gmin.compute_score(sr),
180+
s2 + s3 * 0.1, 0.0, [0, 0, s2, s3 * 0.1], [
181+
{"idx": 0},
182+
{"idx": 1},
183+
{"idx": 2},
184+
{"idx": 3}
185+
])
159186

160187

161188
if __name__ == "__main__":

‎cmstestsuite/unit_tests/grading/scoretypes/GroupMulTest.py‎

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class TestGroupMul(ScoreTypeTestMixin, unittest.TestCase):
3131
def setUp(self):
3232
super().setUp()
3333
self._public_testcases = {
34+
"0_0": True,
3435
"1_0": True,
3536
"1_1": True,
3637
"2_0": True,
@@ -86,8 +87,9 @@ def test_parameter_invalid_testcases_regex_no_match_type(self):
8687
def test_max_scores_regexp(self):
8788
"""Test max score is correct when groups are regexp-defined."""
8889
s1, s2, s3 = 10.5, 30.5, 59
89-
parameters = [[s1, "1_*"], [s2, "2_*"], [s3, "3_*"]]
90-
header = ["Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
90+
parameters = [[0, "0_*"], [s1, "1_*"], [s2, "2_*"], [s3, "3_*"]]
91+
header = ["Subtask 0 (0)",
92+
"Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
9193

9294
# Only group 1_* is public.
9395
public_testcases = dict(self._public_testcases)
@@ -109,8 +111,9 @@ def test_max_scores_regexp(self):
109111
def test_max_scores_number(self):
110112
"""Test max score is correct when groups are number-defined."""
111113
s1, s2, s3 = 10.5, 30.5, 59
112-
parameters = [[s1, 2], [s2, 2], [s3, 2]]
113-
header = ["Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
114+
parameters = [[0, 1], [s1, 2], [s2, 2], [s3, 2]]
115+
header = ["Subtask 0 (0)",
116+
"Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
114117

115118
# Only group 1_* is public.
116119
public_testcases = dict(self._public_testcases)
@@ -131,31 +134,55 @@ def test_max_scores_number(self):
131134

132135
def test_compute_score(self):
133136
s1, s2, s3 = 10.5, 30.5, 59
134-
parameters = [[s1, "1_*"], [s2, "2_*"], [s3, "3_*"]]
137+
parameters = [[0, "0_*"], [s1, "1_*"], [s2, "2_*"], [s3, "3_*"]]
135138
gmul = GroupMul(parameters, self._public_testcases)
136139
sr = self.get_submission_result(self._public_testcases)
137140

138141
# All correct.
139-
self.assertComputeScore(gmul.compute_score(sr),
140-
s1 + s2 + s3, s1, [s1, s2, s3])
142+
self.assertComputeScore(
143+
gmul.compute_score(sr),
144+
s1 + s2 + s3, s1, [0, s1, s2, s3], [
145+
{"idx": 0},
146+
{"idx": 1},
147+
{"idx": 2},
148+
{"idx": 3}
149+
])
141150

142151
# Some non-public subtask is incorrect.
143152
self.set_outcome(sr, "3_1", 0.0)
144-
self.assertComputeScore(gmul.compute_score(sr),
145-
s1 + s2, s1, [s1, s2, 0])
153+
self.assertComputeScore(
154+
gmul.compute_score(sr),
155+
s1 + s2, s1, [0, s1, s2, 0], [
156+
{"idx": 0},
157+
{"idx": 1},
158+
{"idx": 2},
159+
{"idx": 3}
160+
])
146161

147162
# Also the public subtask is incorrect.
148163
self.set_outcome(sr, "1_0", 0.0)
149164
self.set_outcome(sr, "1_1", 0.0)
150-
self.assertComputeScore(gmul.compute_score(sr),
151-
s2, 0.0, [0, s2, 0])
165+
self.assertComputeScore(
166+
gmul.compute_score(sr),
167+
s2, 0.0, [0, 0, s2, 0], [
168+
{"idx": 0},
169+
{"idx": 1},
170+
{"idx": 2},
171+
{"idx": 3}
172+
])
152173

153174
# Some partial results.
154175
self.set_outcome(sr, "3_0", 0.5)
155176
self.set_outcome(sr, "3_1", 0.1)
156-
self.assertComputeScore(gmul.compute_score(sr),
157-
s2 + s3 * 0.5 * 0.1, 0.0,
158-
[0, s2, s3 * 0.5 * 0.1])
177+
self.assertComputeScore(
178+
gmul.compute_score(sr),
179+
s2 + s3 * 0.5 * 0.1, 0.0,
180+
[0, 0, s2, s3 * 0.5 * 0.1], [
181+
{"idx": 0},
182+
{"idx": 1},
183+
{"idx": 2},
184+
{"idx": 3}
185+
])
159186

160187

161188
if __name__ == "__main__":

‎cmstestsuite/unit_tests/grading/scoretypes/GroupThresholdTest.py‎

Lines changed: 41 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class TestGroupThreshold(ScoreTypeTestMixin, unittest.TestCase):
3131
def setUp(self):
3232
super().setUp()
3333
self._public_testcases = {
34+
"0_0": True,
3435
"1_0": True,
3536
"1_1": True,
3637
"2_0": True,
@@ -94,8 +95,9 @@ def test_parameter_invalid_wrong_threshold_type_not_caught(self):
9495
def test_max_scores_regexp(self):
9596
"""Test max score is correct when groups are regexp-defined."""
9697
s1, s2, s3 = 10.5, 30.5, 59
97-
parameters = [[s1, "1_*", 10], [s2, "2_*", 20], [s3, "3_*", 30]]
98-
header = ["Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
98+
parameters = [[0, "0_*", 0], [s1, "1_*", 10], [s2, "2_*", 20], [s3, "3_*", 30]]
99+
header = ["Subtask 0 (0)",
100+
"Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
99101

100102
# Only group 1_* is public.
101103
public_testcases = dict(self._public_testcases)
@@ -120,8 +122,9 @@ def test_max_scores_regexp(self):
120122
def test_max_scores_number(self):
121123
"""Test max score is correct when groups are number-defined."""
122124
s1, s2, s3 = 10.5, 30.5, 59
123-
parameters = [[s1, 2, 10], [s2, 2, 20], [s3, 2, 30]]
124-
header = ["Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
125+
parameters = [[0, 1, 0], [s1, 2, 10], [s2, 2, 20], [s3, 2, 30]]
126+
header = ["Subtask 0 (0)",
127+
"Subtask 1 (10.5)", "Subtask 2 (30.5)", "Subtask 3 (59)"]
125128

126129
# Only group 1_* is public.
127130
public_testcases = dict(self._public_testcases)
@@ -145,33 +148,58 @@ def test_max_scores_number(self):
145148

146149
def test_compute_score(self):
147150
s1, s2, s3 = 10.5, 30.5, 59
148-
parameters = [[s1, "1_*", 10], [s2, "2_*", 20], [s3, "3_*", 30]]
151+
parameters = [[0, "0_*", 0],
152+
[s1, "1_*", 10], [s2, "2_*", 20], [s3, "3_*", 30]]
149153
st = GroupThreshold(parameters, self._public_testcases)
150154
sr = self.get_submission_result(self._public_testcases)
151155

152156
# All correct (below threshold).
153157
for evaluation in sr.evaluations:
154158
evaluation.outcome = 5.5
155-
self.assertComputeScore(st.compute_score(sr),
156-
s1 + s2 + s3, s1, [s1, s2, s3])
159+
self.assertComputeScore(
160+
st.compute_score(sr),
161+
s1 + s2 + s3, s1, [0, s1, s2, s3], [
162+
{"idx": 0},
163+
{"idx": 1},
164+
{"idx": 2},
165+
{"idx": 3}
166+
])
157167

158168
# Some non-public subtask is incorrect.
159169
self.set_outcome(sr, "3_1", 100.5)
160-
self.assertComputeScore(st.compute_score(sr),
161-
s1 + s2, s1, [s1, s2, 0])
170+
self.assertComputeScore(
171+
st.compute_score(sr),
172+
s1 + s2, s1, [0, s1, s2, 0], [
173+
{"idx": 0},
174+
{"idx": 1},
175+
{"idx": 2},
176+
{"idx": 3}
177+
])
162178

163179
# Also the public subtask is incorrect.
164180
self.set_outcome(sr, "1_0", 12.5)
165181
self.set_outcome(sr, "1_1", 12.5)
166-
self.assertComputeScore(st.compute_score(sr),
167-
s2, 0.0, [0, s2, 0])
182+
self.assertComputeScore(
183+
st.compute_score(sr),
184+
s2, 0.0, [0, 0, s2, 0], [
185+
{"idx": 0},
186+
{"idx": 1},
187+
{"idx": 2},
188+
{"idx": 3}
189+
])
168190

169191
# Outcome equal to 0 is special and treated as error even if it is
170192
# below the threshold.
171193
self.set_outcome(sr, "1_0", 0.0)
172194
self.set_outcome(sr, "1_1", 0.0)
173-
self.assertComputeScore(st.compute_score(sr),
174-
s2, 0.0, [0, s2, 0])
195+
self.assertComputeScore(
196+
st.compute_score(sr),
197+
s2, 0.0, [0, 0, s2, 0], [
198+
{"idx": 0},
199+
{"idx": 1},
200+
{"idx": 2},
201+
{"idx": 3}
202+
])
175203

176204

177205
if __name__ == "__main__":

‎cmstestsuite/unit_tests/grading/scoretypes/SumTest.py‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,23 +60,47 @@ def test_compute_score(self):
6060
sr = self.get_submission_result(self._public_testcases)
6161

6262
# All correct.
63-
self.assertComputeScore(st.compute_score(sr),
64-
testcase_score * 4, testcase_score, [])
63+
self.assertComputeScore(
64+
st.compute_score(sr),
65+
testcase_score * 4, testcase_score, [], [
66+
{"idx": '0'},
67+
{"idx": '1'},
68+
{"idx": '2'},
69+
{"idx": '3'}
70+
])
6571

6672
# Some non-public subtask is incorrect.
6773
self.set_outcome(sr, "3", 0.0)
68-
self.assertComputeScore(st.compute_score(sr),
69-
testcase_score * 3, testcase_score, [])
74+
self.assertComputeScore(
75+
st.compute_score(sr),
76+
testcase_score * 3, testcase_score, [], [
77+
{"idx": '0'},
78+
{"idx": '1'},
79+
{"idx": '2'},
80+
{"idx": '3'}
81+
])
7082

7183
# Also the public subtask is incorrect.
7284
self.set_outcome(sr, "1", 0.0)
73-
self.assertComputeScore(st.compute_score(sr),
74-
testcase_score * 2, 0.0, [])
85+
self.assertComputeScore(
86+
st.compute_score(sr),
87+
testcase_score * 2, 0.0, [], [
88+
{"idx": '0'},
89+
{"idx": '1'},
90+
{"idx": '2'},
91+
{"idx": '3'}
92+
])
7593

7694
# Now the public subtask has some partial scores.
7795
self.set_outcome(sr, "1", 0.2)
78-
self.assertComputeScore(st.compute_score(sr),
79-
testcase_score * 2.2, testcase_score * 0.2, [])
96+
self.assertComputeScore(
97+
st.compute_score(sr),
98+
testcase_score * 2.2, testcase_score * 0.2, [], [
99+
{"idx": '0'},
100+
{"idx": '1'},
101+
{"idx": '2'},
102+
{"idx": '3'}
103+
])
80104

81105

82106
if __name__ == "__main__":

‎cmstestsuite/unit_tests/grading/scoretypes/scoretypetestutils.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@
2424
class ScoreTypeTestMixin:
2525
"""A test mixin to make it easier to test score types."""
2626

27-
def assertComputeScore(self, scores, total, public, rws_scores):
27+
def assertComputeScore(self, scores, total, public, rws_scores, subtasks):
2828
self.assertAlmostEqual(scores[0], total)
29+
self.assertEqual([{"idx": s["idx"]} for s in scores[1]],
30+
subtasks)
2931
self.assertAlmostEqual(scores[2], public)
3032
self.assertEqual(scores[4], [str(score) for score in rws_scores])
3133

0 commit comments

Comments
 (0)