Update evaluation for docs

Author: roshambo919

avapi/evaluation/base.py

@@ -1,9 +1,3 @@
-# @Author: Spencer Hallyburton <spencer>
-# @Date:   2022-04-18
-# @Filename: base.py
-# @Last modified by:   spencer
-# @Last modified time: 2022-04-18
-
 import glob
 import os
 from copy import deepcopy
@@ -19,7 +13,7 @@
 )
 from tqdm import tqdm
 
-from avapi.utils import color_from_object_type, get_indices_in_folder
+from avapi.utils import color_from_object_type, get_indices_filenames_in_folder
 
 from ..visualize import snapshot
 from .metrics import precision, recall
@@ -71,7 +65,8 @@ def _results_from_folder(self):
             raise RuntimeError(f"No results to be found in {self.result_path}")
 
         # Get indices by looping
-        self.idxs_available = get_indices_in_folder(glob_dir, self.idxs)
+        # self.idxs_available = get_indices_in_folder(glob_dir, self.idxs)
+        idxs, filenames = get_indices_filenames_in_folder(glob_dir, self.idxs)
 
         # -- run per-frame analysis
         part_func = partial(
@@ -89,15 +84,15 @@ def _results_from_folder(self):
             with Pool(8) as p:
                 res_frame = list(
                     tqdm(
-                        p.imap(part_func, self.idxs_available),
-                        total=len(self.idxs_available),
+                        p.imap(part_func, zip(idxs, filenames)),
+                        total=len(filenames),
                     )
                 )
         else:
-            for idx in tqdm(self.idxs_available):
-                res_frame.append(part_func(idx))
+            for idx, filename in tqdm(zip(idxs, filenames), total=len(filenames)):
+                res_frame.append(part_func((idx, filename)))
 
-        res_frame = {idx: res for idx, res in zip(self.idxs_available, res_frame)}
+        res_frame = {idx: res for idx, res in zip(idxs, res_frame)}
 
         # -- run per-sequence analysis
         res_seq = self._run_per_seq_analysis(res_frame)
@@ -241,9 +236,9 @@ def run_assignment(self, threshold):
             if c in idx_tru_dontcare
         ]
         assigns = {
-            r: c
+            r: c[0]
             for r, c in assignment.iterate_over("rows").items()
-            if list(c.keys())[0] not in idx_tru_dontcare
+            if c[0] not in idx_tru_dontcare
         }
         assignment = OneEdgeBipartiteGraph(
             assigns,
@@ -267,8 +262,7 @@ def run_assignment(self, threshold):
             )
             n_fn = len([idx for idx in idx_FN if self.truths[idx].obj_type == obj_type])
             n_tp = 0
-            for r, cw in assigns.items():
-                c = list(cw.keys())[0]
+            for r, c in assigns.items():
                 if self.detections[r].obj_type == obj_type:
                     if self.truths[c].obj_type == obj_type:
                         n_tp += 1

avapi/evaluation/perception.py

@@ -1,15 +1,12 @@
-# -*- coding: utf-8 -*-
-# @Author: spencer@primus
-# @Date:   2022-06-28
-# @Last Modified by:   spencer@primus
-# @Last Modified time: 2022-09-09
-
 import json
 import os
 
 import numpy as np
 from avstack.geometry import bbox
-from avstack.modules.perception.detections import BoxDetection, DetectionDecoder
+from avstack.modules.perception.detections import (
+    BoxDetection,
+    DetectionContainerDecoder,
+)
 
 from avapi.evaluation.base import ResultAnalyzer, ResultManager
 
@@ -63,11 +60,12 @@ def _run_per_frame_analysis(
         max_dist,
         max_occ,
         whitelist_types,
-        idx,
+        idx_filename,
     ):
         """Load percep status of a single frame"""
+        idx, filename = idx_filename
+
         # Get truths
-        # import ipdb; ipdb.set_trace()
         dist_all = max_dist if sensor_eval_super == "ego" else None
         truths_all = DM.get_objects(
             idx, sensor=sensor_eval_super, max_dist=dist_all, whitelist_types="all"
@@ -85,9 +83,8 @@ def _run_per_frame_analysis(
         )
 
         # Get detections
-        det_file_path = os.path.join(result_path, "%06i.txt" % idx)
-        with open(det_file_path, "rb") as f:
-            detections = json.load(f, cls=DetectionDecoder)
+        with open(filename, "rb") as f:
+            detections = json.load(f, cls=DetectionContainerDecoder)
         dets = [det for det in detections if isinstance(det, (BoxDetection,))]
         metric = (
             "3D_IoU"

avapi/evaluation/tracking.py

@@ -1,9 +1,3 @@
-# -*- coding: utf-8 -*-
-# @Author: spencer@primus
-# @Date:   2022-05-30
-# @Last Modified by:   spencer@primus
-# @Last Modified time: 2022-09-09
-
 import glob
 import logging
 import os
@@ -70,9 +64,11 @@ def _run_per_frame_analysis(
         max_dist,
         max_occ,
         whitelist_types,
-        idx,
+        idx_filename,
     ):
         """Load track status of a single frame"""
+        idx, filename = idx_filename
+
         # Get truths
         dist_all = max_dist if sensor_eval_super == "ego" else None
         truths_all = DM.get_objects(
@@ -91,10 +87,8 @@ def _run_per_frame_analysis(
         )
 
         # Get tracks
-        trk_file_path = os.path.join(result_path, "%06i.txt" % idx)
-        tracks = DM.get_objects_from_file(
-            trk_file_path, whitelist_types=whitelist_types
-        )
+        # trk_file_path = os.path.join(result_path, "%06i.txt" % idx)
+        tracks = DM.get_objects_from_file(filename, whitelist_types=whitelist_types)
 
         # -- 2d analysis
 
@@ -142,6 +136,7 @@ def _run_expanded_analysis(
         DM, save_folder, sensor_eval, sensor_eval_super, whitelist_types, max_dist
     ):
         """Run HOTA metrics"""
+        return {}
         tracker_name = "no-name"
 
         # --- metrics evaluator

avapi/utils.py

@@ -29,6 +29,33 @@ def remove_glob(glob_files):
         print("Removed files from: {}".format(os.path.dirname(f)), flush=True)
 
 
+def get_indices_filenames_in_folder(glob_dir, idxs=None):
+    idxs_available = []
+    fnames_available = []
+    for f in glob_dir:
+        if "log" in f:
+            continue
+        try:
+            idx = int(f.split("/")[-1].replace(".txt", ""))
+        except ValueError:
+            idx = int(f.split("/")[-1].split("-")[2])
+        if idxs is not None:
+            try:
+                iterator = iter(idxs)
+            except TypeError:
+                # not iterable
+                if not (idx == idxs):
+                    continue
+            else:
+                # iterable
+                if idx not in idxs:
+                    continue
+        if idx not in idxs_available:
+            idxs_available.append(idx)
+            fnames_available.append(f)
+    return idxs_available, fnames_available
+
+
 def get_indices_in_folder(glob_dir, idxs=None):
     """Get indices of items in a glob_dir
     optionally: enforce that they are in the list idxs
@@ -37,7 +64,10 @@ def get_indices_in_folder(glob_dir, idxs=None):
     for f in glob_dir:
         if "log" in f:
             continue
-        idx = int(f.split("/")[-1].replace(".txt", ""))
+        try:
+            idx = int(f.split("/")[-1].replace(".txt", ""))
+        except ValueError:
+            idx = int(f.split("/")[-1].split("-")[2])
         if idxs is not None:
             try:
                 iterator = iter(idxs)

avapi/visualize/replay.py

@@ -12,13 +12,12 @@
     print("Cannot find ipywidgets...cannot run visualizations")
 
 from avstack.environment.objects import VehicleState
-from avstack.geometry import bbox
+from avstack.geometry import Attitude, GlobalOrigin3D, Position, bbox
 from avstack.modules.perception.detections import BoxDetection
 
-from .snapshot import show_image_with_boxes
-
+from avapi.utils import color_from_object_type
 
-# from avapi.evaluation import ResultManager, color_from_object_type
+from .snapshot import show_image_with_boxes
 
 
 # ========================================================
@@ -91,6 +90,8 @@ def load_ground_truth_data(folder):
 
 
 def replay_ground_truth_from_folder(folder, viz_type="track"):
+    from avapi.evaluation import ResultManager
+
     assert viz_type in ["track", "track_percep"]
     print("Replaying ground truth data from {}".format(folder))
     ego_data, npc_data = load_ground_truth_data(folder)
@@ -120,6 +121,8 @@ def replay_ground_truth_from_folder(folder, viz_type="track"):
 
 
 def replay_ground_truth_from_data_manager(DM, sensor="main_camera"):
+    from avapi.evaluation import ResultManager
+
     ego_data = [DM.get_ego(frame) for frame in DM.frames]
     npc_data = [DM.get_objects(frame) for frame in DM.frames]
 
@@ -197,7 +200,7 @@ def replay_track_results(
             add_points_real.append([])
             for pt in pts:
                 if (pt is not None) and (not isinstance(pt, str)):
-                    pt.change_origin(NominalOriginStandard)
+                    pt.change_reference(GlobalOrigin3D, inplace=True)
                     add_points_real[i].append(pt)
         add_points = add_points_real
     if (highlight_track_IDs is None) or (len(highlight_track_IDs) == 0):
@@ -206,7 +209,7 @@ def replay_track_results(
         assert len(highlight_track_IDs) == len(track_results), len(highlight_track_IDs)
 
     if ego_box is not None:
-        ego_box.change_origin(NominalOriginStandard)
+        ego_box.change_reference(GlobalOrigin3D, inplace=True)
 
     # Get all track locations and predictions ahead of time
     trk_points = {}  # ID: frame: pt
@@ -216,7 +219,7 @@ def replay_track_results(
     for idx, tr in track_results.items():
         for track in tr["result"].tracks:
             if isinstance(track, VehicleState):
-                track.change_origin(NominalOriginStandard)
+                track.change_reference(GlobalOrigin3D, inplace=True)
                 if track.ID not in trk_points:
                     trk_points[track.ID] = {}
                     trk_preds[track.ID] = {}
@@ -229,7 +232,7 @@ def replay_track_results(
                     raise NotImplementedError(type(track))
         for truth in tr["result"].truths:
             if isinstance(truth, VehicleState):
-                truth.change_origin(NominalOriginStandard)
+                truth.change_reference(GlobalOrigin3D, inplace=True)
 
     def f(idx):
         axs_slider.clear()
@@ -255,14 +258,9 @@ def plot_tracks(colors, tracks):
                         if do_highlight:
                             fac = 1.2
                             box_enlarged = bbox.Box3D(
-                                [
-                                    fac * track.box.h,
-                                    fac * track.box.w,
-                                    fac * track.box.l,
-                                    track.position,
-                                    track.box.attitude,
-                                ],
-                                track.position.origin,
+                                position=track.position,
+                                attitude=track.attitude,
+                                hwl=[fac * d for d in track.box.size],
                             )
                             axs_slider.add_patch(
                                 _box_to_bev_rect(
@@ -361,7 +359,7 @@ def plot_tracks(colors, tracks):
             ):
                 col = tuple(c / 255.0 for c in color)
                 if "bev" in projection:
-                    truth.box3d.change_origin(NominalOriginStandard)
+                    truth.box3d.change_reference(GlobalOrigin3D, inplace=True)
                     rect = _box_to_bev_rect(
                         truth.box3d,
                         col,
@@ -376,7 +374,9 @@ def plot_tracks(colors, tracks):
 
         # add fake elements to legend
         nominal_box = bbox.Box3D(
-            [2, 2, 4, np.zeros((3,)), np.quaternion(1)], NominalOriginStandard
+            position=Position(np.zeros((3,)), GlobalOrigin3D),
+            attitude=Attitude(np.quaternion(1), GlobalOrigin3D),
+            hwl=[2, 2, 4],
         )
         # -- unsafe
         handles.append(
@@ -535,12 +535,12 @@ def replay_track_percep_results(
             track_boxes = []
             for trk in track_results[idx]["result"].tracks:
                 trk_copy = deepcopy(trk).as_object()
-                trk_copy.change_origin(calib.origin)
+                trk_copy.change_reference(calib.reference, inplace=True)
                 track_boxes.append(trk_copy)
             box_colors = track_results[idx]["result"].colors["detections"]
             if show_truth:
                 for truth in track_results[idx]["result"].truths:
-                    truth.box.change_origin(calib.origin)
+                    truth.box.change_reference(calib.reference, inplace=True)
                     track_boxes.append(truth)
                 box_colors.extend(track_results[idx]["result"].colors["truths"])
         else:
@@ -553,7 +553,10 @@ def replay_track_percep_results(
         if "fv" in projection:
             img = DM.get_image(idx, sensor=sensor)
             img3d = show_image_with_boxes(
-                img, track_boxes, box_colors=box_colors, show=False, return_images=True
+                img,
+                track_boxes,
+                box_colors=box_colors,
+                show=False,
             )
             imgs_ALL["fv"] = (
                 img3d[..., None]
@@ -612,7 +615,7 @@ def _box_to_bev_rect(
         (left, bottom),
         width,
         height,
-        angle,
+        angle=angle,
         alpha=0.9,
         edgecolor=color,
         facecolor=facecolor,