From 52ce2a4be08a80a6248c29da1f8297a793074961 Mon Sep 17 00:00:00 2001
From: Leonard Lausen <lausen@amazon.com>
Date: Thu, 7 Jan 2021 23:11:09 +0000
Subject: [PATCH] Enable bias correction in AdamW when fine-tuning BERT

Mosbach, Marius, Maksym Andriushchenko, and Dietrich Klakow. "On the Stability of Fine-tuning BERT: Misconceptions, Explanations, and Strong Baselines." arXiv preprint arXiv:2006.04884 (2020).

Zhang, Tianyi, et al. "Revisiting Few-sample BERT Fine-tuning." arXiv preprint arXiv:2006.05987 (2020).
---
 scripts/question_answering/run_squad.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/question_answering/run_squad.py b/scripts/question_answering/run_squad.py
index 521ee15a47..516d5b8edb 100644
--- a/scripts/question_answering/run_squad.py
+++ b/scripts/question_answering/run_squad.py
@@ -543,7 +543,7 @@ def train(args):
         optimizer_params.update({'beta1': adam_betas[0],
                                  'beta2': adam_betas[1],
                                  'epsilon': args.adam_epsilon,
-                                 'correct_bias': False,
+                                 'correct_bias': True,
                                  })
     elif args.optimizer == 'adam':
         optimizer_params.update({'beta1': adam_betas[0],