Adapt task_score() to also generate contestant-appropriate info

A contestant can see two task_score: public and tokened. Public looks
at the public score for all submissions; tokened looks at the total
score for tokened submissions.

Note that we cannot handle the tokened case restricting the list of
submissions, because that could change the last submission, which is
used in some scoremodes.

Author: stefano-maggiolo

cms/grading/scoring.py

@@ -109,12 +109,19 @@ def compare(a, b):
 
 # Computing global scores (for ranking).
 
-def task_score(participation, task):
+def task_score(participation, task, public=False, only_tokened=False):
     """Return the score of a contest's user on a task.
 
     participation (Participation): the user and contest for which to
         compute the score.
     task (Task): the task for which to compute the score.
+    public (bool): if True, compute the public score (that is, the one
+        discoverable looking only at the results of public testcases) instead
+        of the full score.
+    only_tokened (bool): if True, compute the score discoverable only looking
+        at the results of tokened submissions (that is, the score that the user
+        would obtain if all non-tokened submissions scored 0.0, or equivalently
+        had not been scored yet).
 
     return ((float, bool)): the score of user on task, and True if not
         all submissions of the participation in the task have been scored.
@@ -128,6 +135,12 @@ def task_score(participation, task):
     # submission_results table. Doing so means that this function should incur
     # no exta database queries.
 
+    if public and only_tokened:
+        raise ValueError(
+            "Requested public task score restricted to tokened submissions. "
+            "This is a programming error: users have access to all public "
+            "scores regardless of token status.")
+
     submissions = [s for s in participation.submissions
                    if s.task is task and s.official]
     if len(submissions) == 0:
@@ -137,54 +150,67 @@ def task_score(participation, task):
         (s, s.get_result(task.active_dataset))
         for s in sorted(submissions, key=lambda s: s.timestamp)]
 
+    score_details_tokened = []
+    partial = False
+    for s, sr in submissions_and_results:
+        if sr is None or not sr.scored():
+            partial = True
+            score, score_details = None, None
+        elif public:
+            score, score_details = sr.public_score, sr.public_score_details
+        elif only_tokened and not s.tokened():
+            # If the caller wants the only_tokened score and this submission is
+            # not tokened, the score mode should ignore its score. To do so, we
+            # send to the score mode what we would send if it wasn't already
+            # scored.
+            score, score_details = None, None
+        else:
+            score, score_details = sr.score, sr.score_details
+        score_details_tokened.append((score, score_details, s.tokened()))
+
     if task.score_mode == SCORE_MODE_MAX:
-        return _task_score_max(submissions_and_results)
+        return _task_score_max(score_details_tokened), partial
     if task.score_mode == SCORE_MODE_MAX_SUBTASK:
-        return _task_score_max_subtask(submissions_and_results)
+        return _task_score_max_subtask(score_details_tokened), partial
     elif task.score_mode == SCORE_MODE_MAX_TOKENED_LAST:
-        return _task_score_max_tokened_last(submissions_and_results)
+        return _task_score_max_tokened_last(score_details_tokened), partial
     else:
         raise ValueError("Unknown score mode '%s'" % task.score_mode)
 
 
-def _task_score_max_tokened_last(submissions_and_results):
+def _task_score_max_tokened_last(score_details_tokened):
     """Compute score using the "max tokened last" score mode.
 
     This was used in IOI 2010-2012. The score of a participant on a task is
     the maximum score amongst all tokened submissions and the last submission
     (not yet computed scores count as 0.0).
 
-    submissions_and_results ([(Submission, SubmissionResult|None)]): list of
-        all submissions and their results for the participant on the task (on
-        the dataset of interest); result is None if not available (that is,
-        if the submission has not been compiled).
+    score_details_tokened ([(float|None, object|None, bool)]): a tuple for each
+        submission of the user in the task, containing score, score details
+        (each None if not scored yet) and if the submission was tokened.
 
-    return ((float, bool)): (score, partial), same as task_score().
+    return (float): the score.
 
     """
-    partial = False
 
     # The score of the last submission (if computed, otherwise 0.0). Note that
     # partial will be set to True in the next loop.
-    last_score = 0.0
-    _, last_sr = submissions_and_results[-1]
-    if last_sr is not None and last_sr.scored():
-        last_score = last_sr.score
+    last_score, _, _ = score_details_tokened[-1]
+    if last_score is None:
+        last_score = 0.0
 
     # The maximum score amongst the tokened submissions (not yet computed
     # scores count as 0.0).
     max_tokened_score = 0.0
-    for s, sr in submissions_and_results:
-        if sr is not None and sr.scored():
-            if s.tokened():
-                max_tokened_score = max(max_tokened_score, sr.score)
-        else:
-            partial = True
+    for score, _, tokened in score_details_tokened:
+        if score is not None:
+            if tokened:
+                max_tokened_score = max(max_tokened_score, score)
 
-    return max(last_score, max_tokened_score), partial
+    return max(last_score, max_tokened_score)
 
 
-def _task_score_max_subtask(submissions_and_results):
+def _task_score_max_subtask(score_details_tokened):
     """Compute score using the "max subtask" score mode.
 
     This has been used in IOI since 2017. The score of a participant on a
@@ -196,68 +222,60 @@ def _task_score_max_subtask(submissions_and_results):
     this is not true, the score mode will work as if the task had a single
     subtask.
 
-    submissions_and_results ([(Submission, SubmissionResult|None)]): list of
-        all submissions and their results for the participant on the task (on
-        the dataset of interest); result is None if not available (that is,
-        if the submission has not been compiled).
+    score_details_tokened ([(float|None, object|None, bool)]): a tuple for each
+        submission of the user in the task, containing score, score details
+        (each None if not scored yet) and if the submission was tokened.
 
-    return ((float, bool)): (score, partial), same as task_score().
+    return (float): the score.
 
     """
     # Maximum score for each subtask (not yet computed scores count as 0.0).
     max_scores = {}
 
-    partial = False
-    for _, sr in submissions_and_results:
-        if sr is None or not sr.scored():
-            partial = True
+    for score, details, _ in score_details_tokened:
+        if score is None:
             continue
 
-        if sr.score_details == [] and sr.score == 0.0:
+        if details == [] and score == 0.0:
             # Submission did not compile, ignore it.
             continue
 
         try:
             subtask_scores = dict(
                 (subtask["idx"],
                  subtask["score_fraction"] * subtask["max_score"])
-                for subtask in sr.score_details
-            )
+                for subtask in details)
         except Exception:
             subtask_scores = None
 
         if subtask_scores is None or len(subtask_scores) == 0:
             # Task's score type is not group, assume a single subtask.
-            subtask_scores = {1: sr.score}
+            subtask_scores = {1: score}
 
         for idx, score in iteritems(subtask_scores):
             max_scores[idx] = max(max_scores.get(idx, 0.0), score)
 
-    return sum(itervalues(max_scores)), partial
+    return sum(itervalues(max_scores))
 
 
-def _task_score_max(submissions_and_results):
+def _task_score_max(score_details_tokened):
     """Compute score using the "max" score mode.
 
     This was used in IOI 2013-2016. The score of a participant on a task is
     the maximum score amongst all submissions (not yet computed scores count
     as 0.0).
 
-    submissions_and_results ([(Submission, SubmissionResult|None)]): list of
-        all submissions and their results for the participant on the task (on
-        the dataset of interest); result is None if not available (that is,
-        if the submission has not been compiled).
+    score_details_tokened ([(float|None, object|None, bool)]): a tuple for each
+        submission of the user in the task, containing score, score details
+        (each None if not scored yet) and if the submission was tokened.
 
-    return ((float, bool)): (score, partial), same as task_score().
+    return (float): the score.
 
     """
-    partial = False
-    score = 0.0
+    max_score = 0.0
 
-    for _, sr in submissions_and_results:
-        if sr is not None and sr.scored():
-            score = max(score, sr.score)
-        else:
-            partial = True
+    for score, _, _ in score_details_tokened:
+        if score is not None:
+            max_score = max(max_score, score)
 
-    return score, partial
+    return max_score

cmstestsuite/unit_tests/grading/scoring_test.py

@@ -54,10 +54,15 @@ def setUp(self):
     def at(self, timestamp):
         return self.timestamp + timedelta(seconds=timestamp)
 
-    def call(self):
-        return task_score(self.participation, self.task)
-
-    def add_result(self, timestamp, score, tokened=False, score_details=None):
+    def call(self, public=False, only_tokened=False):
+        return task_score(self.participation, self.task,
+                          public=public, only_tokened=only_tokened)
+
+    def add_result(self, timestamp, score, tokened=False, score_details=None,
+                   public_score=None, public_score_details=None):
+        public_score = public_score if public_score is not None else 0.0
+        public_score_details = public_score_details \
+            if public_score_details is not None else []
         score_details = score_details if score_details is not None else []
         submission = self.add_submission(
             participation=self.participation,
@@ -67,9 +72,9 @@ def add_result(self, timestamp, score, tokened=False, score_details=None):
         # must be set to declare the submission result as scored.
         self.add_submission_result(submission, self.task.active_dataset,
                                    score=score,
-                                   public_score=score,
+                                   public_score=public_score,
                                    score_details=score_details,
-                                   public_score_details=score_details,
+                                   public_score_details=public_score_details,
                                    ranking_score_details=[])
         if tokened:
             self.add_token(timestamp=timestamp, submission=submission)
@@ -149,6 +154,20 @@ def test_all_unscored(self):
         self.session.flush()
         self.assertEqual(self.call(), (0.0, True))
 
+    def test_public(self):
+        self.add_result(self.at(1), 44.4, tokened=True, public_score=4.4)
+        self.add_result(self.at(2), 66.6, tokened=False, public_score=66.6)
+        self.add_result(self.at(3), 11.1, tokened=False, public_score=11.1)
+        self.session.flush()
+        self.assertEqual(self.call(public=True), (11.1, False))
+
+    def test_only_tokened(self):
+        self.add_result(self.at(1), 11.1, tokened=True)
+        self.add_result(self.at(2), 66.6, tokened=False)
+        self.add_result(self.at(3), 44.4, tokened=False)
+        self.session.flush()
+        self.assertEqual(self.call(only_tokened=True), (11.1, False))
+
 
 class TestTaskScoreMaxSubtask(TaskScoreMixin, unittest.TestCase):
     """Tests for task_score() using the max_subtask score mode."""
@@ -247,6 +266,60 @@ def test_rounding(self):
         self.session.flush()
         self.assertEqual(self.call(), (80 + 0.0004, False))
 
+    def test_public(self):
+        self.add_result(self.at(1),
+                        30 * 1.0 + 40 * 1.0 + 30 * 1.0,
+                        score_details=[
+                            self.subtask(3, 30, 1.0),
+                            self.subtask(2, 40, 1.0),
+                            self.subtask(1, 30, 1.0),
+                        ],
+                        public_score=30 * 0.2 + 40 * 0.5 + 30 * 0.1,
+                        public_score_details=[
+                            self.subtask(3, 30, 0.2),
+                            self.subtask(2, 40, 0.5),
+                            self.subtask(1, 30, 0.1),
+                        ])
+        self.add_result(self.at(2),
+                        30 * 1.0 + 40 * 1.0 + 30 * 1.0,
+                        score_details=[
+                            self.subtask(2, 40, 1.0),
+                            self.subtask(1, 30, 1.0),
+                            self.subtask(3, 30, 1.0),
+                        ],
+                        public_score=30 * 0.1 + 40 * 0.5 + 30 * 0.2,
+                        public_score_details=[
+                            self.subtask(2, 40, 0.5),
+                            self.subtask(1, 30, 0.2),
+                            self.subtask(3, 30, 0.1),
+                        ])
+        self.session.flush()
+        self.assertEqual(self.call(public=True),
+                         (30 * 0.2 + 40 * 0.5 + 30 * 0.2, False))
+
+    def test_only_tokened(self):
+        self.add_result(self.at(1), 30 * 0.2 + 40 * 0.5 + 30 * 0.1,
+                        score_details=[
+                            self.subtask(3, 30, 0.2),
+                            self.subtask(2, 40, 0.5),
+                            self.subtask(1, 30, 0.1),
+                        ], tokened=True)
+        self.add_result(self.at(2), 30 * 0.1 + 40 * 0.5 + 30 * 0.2,
+                        score_details=[
+                            self.subtask(2, 40, 0.5),
+                            self.subtask(1, 30, 0.2),
+                            self.subtask(3, 30, 0.1),
+                        ], tokened=True)
+        self.add_result(self.at(3), 30 * 1.0 + 40 * 1.0 + 30 * 1.0,
+                        score_details=[
+                            self.subtask(2, 40, 1.0),
+                            self.subtask(1, 30, 1.0),
+                            self.subtask(3, 30, 1.0),
+                        ], tokened=False)
+        self.session.flush()
+        self.assertEqual(self.call(only_tokened=True),
+                         (30 * 0.2 + 40 * 0.5 + 30 * 0.2, False))
+
 
 class TestTaskScoreMax(TaskScoreMixin, unittest.TestCase):
     """Tests for task_score() using the max score mode."""
@@ -290,6 +363,18 @@ def test_all_unscored(self):
         self.session.flush()
         self.assertEqual(self.call(), (0.0, True))
 
+    def test_public(self):
+        self.add_result(self.at(1), 44.4, tokened=False, public_score=44.4)
+        self.add_result(self.at(2), 66.6, tokened=False, public_score=6.6)
+        self.session.flush()
+        self.assertEqual(self.call(public=True), (44.4, False))
+
+    def test_only_tokened(self):
+        self.add_result(self.at(1), 44.4, tokened=True)
+        self.add_result(self.at(2), 66.6, tokened=False)
+        self.session.flush()
+        self.assertEqual(self.call(only_tokened=True), (44.4, False))
+
 
 if __name__ == "__main__":
     unittest.main()