Add fps info to samplers benchmark (#294)

benchmarks/samplers/benchmark_samplers.py

@@ -16,16 +16,22 @@ def bench(f, *args, num_exp=100, warmup=0, **kwargs):
     for _ in range(warmup):
         f(*args, **kwargs)
 
+    num_frames = None
     times = []
     for _ in range(num_exp):
         start = perf_counter_ns()
-        f(*args, **kwargs)
+        clips = f(*args, **kwargs)
         end = perf_counter_ns()
         times.append(end - start)
-    return torch.tensor(times).float()
+        num_frames = (
+            clips.data.shape[0] * clips.data.shape[1]
+        )  # should be constant across calls
+    return torch.tensor(times).float(), num_frames
+
 
+def report_stats(times, num_frames, unit="ms"):
+    fps = num_frames * 1e9 / torch.median(times)
 
-def report_stats(times, unit="ms"):
     mul = {
         "ns": 1,
         "µs": 1e-3,
@@ -35,13 +41,13 @@ def report_stats(times, unit="ms"):
     times = times * mul
     std = times.std().item()
     med = times.median().item()
-    print(f"{med = :.2f}{unit} +- {std:.2f}")
-    return med
+    print(f"{med = :.2f}{unit} +- {std:.2f}  med fps = {fps:.1f}")
+    return med, fps
 
 
 def sample(sampler, **kwargs):
     decoder = VideoDecoder(VIDEO_PATH)
-    sampler(
+    return sampler(
         decoder,
         num_frames_per_clip=10,
         **kwargs,
@@ -56,34 +62,34 @@ def sample(sampler, **kwargs):
     print(f"{num_clips = }")
 
     print("clips_at_random_indices     ", end="")
-    times = bench(
+    times, num_frames = bench(
         sample, clips_at_random_indices, num_clips=num_clips, num_exp=NUM_EXP, warmup=2
     )
-    report_stats(times, unit="ms")
+    report_stats(times, num_frames, unit="ms")
 
     print("clips_at_regular_indices    ", end="")
-    times = bench(
+    times, num_frames = bench(
         sample, clips_at_regular_indices, num_clips=num_clips, num_exp=NUM_EXP, warmup=2
     )
-    report_stats(times, unit="ms")
+    report_stats(times, num_frames, unit="ms")
 
     print("clips_at_random_timestamps  ", end="")
-    times = bench(
+    times, num_frames = bench(
         sample,
         clips_at_random_timestamps,
         num_clips=num_clips,
         num_exp=NUM_EXP,
         warmup=2,
     )
-    report_stats(times, unit="ms")
+    report_stats(times, num_frames, unit="ms")
 
     print("clips_at_regular_timestamps ", end="")
     seconds_between_clip_starts = 13 / num_clips  # approximate. video is 13s long
-    times = bench(
+    times, num_frames = bench(
         sample,
         clips_at_regular_timestamps,
         seconds_between_clip_starts=seconds_between_clip_starts,
         num_exp=NUM_EXP,
         warmup=2,
     )
-    report_stats(times, unit="ms")
+    report_stats(times, num_frames, unit="ms")