Merge branch 'main' into feature/fix-optuna-v3

.circleci/config.yml

@@ -131,7 +131,7 @@ jobs:
             conda activate hydra
             pip install nox dataclasses --progress-bar off
             if [ -n "$CIRCLE_PR_NUMBER" ]; then
-              SKIP_PLUGINS=hydra_ray_launcher nox -s lint test_tools test_core \
+              nox -s lint test_tools test_core \
                 test_jupyter_notebooks lint_plugins test_plugins test_plugins_vs_core -ts
             else
               nox -s lint test_tools test_core test_jupyter_notebooks -ts
@@ -152,7 +152,7 @@ jobs:
             export NOX_PYTHON_VERSIONS=<< parameters.py_version >>
             pip install nox dataclasses --progress-bar off
             if [ -n "$CIRCLE_PR_NUMBER" ]; then
-              SKIP_PLUGINS=hydra_ray_launcher nox -s lint test_tools test_core \
+              nox -s lint test_tools test_core \
                 test_jupyter_notebooks lint_plugins test_plugins test_plugins_vs_core -ts
             else
               nox -s lint test_tools test_core test_jupyter_notebooks -ts
@@ -174,7 +174,6 @@ jobs:
             $env:PYTHONIOENCODING="utf_8"
             conda activate hydra
             If ($env:CIRCLE_PR_NUMBER) {
-              $env:SKIP_PLUGINS="hydra_ray_launcher"
               nox -s lint test_tools test_core test_jupyter_notebooks lint_plugins test_plugins test_plugins_vs_core -ts
             } else {
               nox -s lint test_tools test_core test_jupyter_notebooks -ts

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,7 +4,7 @@
 
 (Write your motivation for proposed changes here.)
 
-### Have you read the [Contributing Guidelines on pull requests](https://github.com/facebookresearch/hydra/blob/master/CONTRIBUTING.md)?
+### Have you read the [Contributing Guidelines on pull requests](https://github.com/facebookresearch/hydra/blob/main/CONTRIBUTING.md)?
 
 Yes/No
 

.github/workflows/deploy_website.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Deploy website
       env:
         WEBSITE_GITHUB_TOKEN: ${{ secrets.WEBSITE_GITHUB_TOKEN }}

.mypy.ini

@@ -11,3 +11,9 @@ ignore_missing_imports = True
 [mypy-hydra.grammar.gen.*]
 ignore_missing_imports = True
 ignore_errors = True
+
+[mypy-nevergrad.*]
+ignore_missing_imports = True
+
+[mypy-rq.*]
+ignore_missing_imports = True

README.md

@@ -1,4 +1,4 @@
-<p align="center"><img src="https://raw.githubusercontent.com/facebookresearch/hydra/master/website/static/img/Hydra-Readme-logo2.svg" alt="logo" width="70%" /></p>
+<p align="center"><img src="https://raw.githubusercontent.com/facebookresearch/hydra/main/website/static/img/Hydra-Readme-logo2.svg" alt="logo" width="70%" /></p>
 
 <p align="center">
   <a href="https://pypi.org/project/hydra-core/">

build_helpers/test_helpers.py

@@ -112,8 +112,8 @@ def test_find(
         scan_exclude=scan_exclude,
     )
 
-    ret_set = set([str(Path(x)) for x in ret])
-    expected_set = set([str(Path(x)) for x in expected])
+    ret_set = {str(Path(x)) for x in ret}
+    expected_set = {str(Path(x)) for x in expected}
     assert ret_set == expected_set
 
 

examples/plugins/example_configsource_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import find_namespace_packages, setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-configsource",
@@ -19,7 +19,6 @@
             "License :: OSI Approved :: MIT License",
             # Hydra uses Python version and Operating system to determine
             # In which environments to test this plugin
-            "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",

examples/plugins/example_generic_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import find_namespace_packages, setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-generic-plugin",
@@ -19,7 +19,6 @@
             "License :: OSI Approved :: MIT License",
             # Hydra uses Python version and Operating system to determine
             # In which environments to test this plugin
-            "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",

examples/plugins/example_launcher_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import find_namespace_packages, setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-launcher",
@@ -19,7 +19,6 @@
             "License :: OSI Approved :: MIT License",
             # Hydra uses Python version and Operating system to determine
             # In which environments to test this plugin
-            "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",

examples/plugins/example_registered_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
 setup(
     name="hydra-example-registered-plugin",
@@ -19,7 +19,6 @@
         "License :: OSI Approved :: MIT License",
         # Hydra uses Python version and Operating system to determine
         # In which environments to test this plugin
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",

examples/plugins/example_searchpath_plugin/setup.py

@@ -3,7 +3,7 @@
 from setuptools import find_namespace_packages, setup, find_packages
 
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-searchpath-plugin",
@@ -21,7 +21,6 @@
             "License :: OSI Approved :: MIT License",
             # Hydra uses Python version and Operating system to determine
             # In which environments to test this plugin
-            "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",

examples/plugins/example_sweeper_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import find_namespace_packages, setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-sweeper",
@@ -19,7 +19,6 @@
             "License :: OSI Approved :: MIT License",
             # Hydra uses Python version and Operating system to determine
             # In which environments to test this plugin
-            "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",

examples/tutorials/basic/running_your_hydra_app/3_working_directory/my_app.py

@@ -9,6 +9,9 @@
 @hydra.main(version_base=None)
 def my_app(_cfg: DictConfig) -> None:
     print(f"Working directory : {os.getcwd()}")
+    print(
+        f"Output directory  : {hydra.core.hydra_config.HydraConfig.get().runtime.output_dir}"
+    )
 
 
 if __name__ == "__main__":

hydra/_internal/core_plugins/basic_sweeper.py

@@ -62,7 +62,7 @@ def __init__(
         """
         Instantiates
         """
-        super(BasicSweeper, self).__init__()
+        super().__init__()
 
         if params is None:
             params = {}

hydra/_internal/core_plugins/zsh_completion.py

@@ -10,7 +10,7 @@
 
 class ZshCompletion(CompletionPlugin):
     def __init__(self, config_loader: ConfigLoader):
-        super(ZshCompletion, self).__init__(config_loader)
+        super().__init__(config_loader)
         from hydra._internal.core_plugins.bash_completion import BashCompletion
 
         self.delegate = BashCompletion(config_loader)

hydra/_internal/defaults_list.py

@@ -312,7 +312,7 @@ def _check_not_missing(
                 group_path,
                 results_filter=ObjectType.CONFIG,
             )
-            opt_list = "\n".join(["\t" + x for x in options])
+            opt_list = "\n".join("\t" + x for x in options)
             msg = dedent(
                 f"""\
                 You must specify '{override_key}', e.g, {override_key}=<OPTION>
@@ -778,7 +778,7 @@ def config_not_found_error(repo: IConfigRepository, tree: DefaultsTreeNode) -> N
         if isinstance(element, GroupDefault):
             msg = f"Could not find '{element.get_config_path()}'\n"
             if options is not None and len(options) > 0:
-                opt_list = "\n".join(["\t" + x for x in options])
+                opt_list = "\n".join("\t" + x for x in options)
                 msg = f"{msg}\nAvailable options in '{group}':\n" + opt_list
         else:
             msg = dedent(

hydra/_internal/grammar/grammar_functions.py

@@ -15,7 +15,7 @@
     Sweep,
 )
 
-ElementType = Union[str, int, bool, float, list, dict]
+ElementType = Union[str, int, bool, float, List[Any], Dict[str, Any]]
 
 
 def apply_to_dict_values(

hydra/_internal/hydra.py

@@ -233,7 +233,7 @@ def get_shell_to_plugin_map(
 
         for shell, plugins in shell_to_plugin.items():
             if len(plugins) > 1:
-                lst = ",".join([type(plugin).__name__ for plugin in plugins])
+                lst = ",".join(type(plugin).__name__ for plugin in plugins)
                 raise ValueError(f"Multiple plugins installed for {shell} : {lst}")
 
         return shell_to_plugin
@@ -251,7 +251,7 @@ def shell_completion(
 
         def find_plugin(cmd: str) -> CompletionPlugin:
             if cmd not in shell_to_plugin:
-                lst = "\n".join(["\t" + x for x in shell_to_plugin.keys()])
+                lst = "\n".join("\t" + x for x in shell_to_plugin.keys())
                 raise ValueError(
                     f"No completion plugin for '{cmd}' found, available : \n{lst}"
                 )
@@ -287,7 +287,7 @@ def list_all_config_groups(self, parent: str = "") -> Sequence[str]:
             if parent == "":
                 group_name = group
             else:
-                group_name = "{}/{}".format(parent, group)
+                group_name = f"{parent}/{group}"
             files = self.config_loader.get_group_options(group_name, ObjectType.CONFIG)
             dirs = self.config_loader.get_group_options(group_name, ObjectType.GROUP)
             if len(files) > 0:
@@ -305,10 +305,10 @@ def format_config_groups(
             options = sorted(self.config_loader.get_group_options(group))
             if compact:
                 items = ", ".join(options)
-                line = "{}: {}".format(group, items)
+                line = f"{group}: {items}"
             else:
-                items = "\n".join(["  " + o for o in options])
-                line = "{}:\n{}".format(group, items)
+                items = "\n".join("  " + o for o in options)
+                line = f"{group}:\n{items}"
             s += line + "\n"
 
         return s
@@ -396,14 +396,14 @@ def _print_plugins(self) -> None:
             if len(plugins) > 0:
                 Hydra._log_header(header=f"{plugin_type.__name__}:", prefix="\t")
                 for plugin in plugins:
-                    log.debug("\t\t{}".format(plugin.__name__))
+                    log.debug(f"\t\t{plugin.__name__}")
                     if plugin.__name__ in all_plugins:
                         all_plugins.remove(plugin.__name__)
 
         if len(all_plugins) > 0:
             Hydra._log_header(header="Generic plugins: ", prefix="\t")
             for plugin_name in all_plugins:
-                log.debug("\t\t{}".format(plugin_name))
+                log.debug(f"\t\t{plugin_name}")
 
     def _print_search_path(
         self,

hydra/_internal/utils.py

@@ -494,7 +494,7 @@ def _get_completion_help() -> str:
             completion_info.append(plugin_cls.help(cmd).format(_get_exec_command()))
         completion_info.append("")
 
-    completion_help = "\n".join([f"    {x}" if x else x for x in completion_info])
+    completion_help = "\n".join(f"    {x}" if x else x for x in completion_info)
     return completion_help
 
 

hydra/core/config_store.py

@@ -133,10 +133,10 @@ def get_type(self, path: str) -> ObjectType:
     def list(self, path: str) -> List[str]:
         d = self._open(path)
         if d is None:
-            raise IOError(f"Path not found {path}")
+            raise OSError(f"Path not found {path}")
 
         if not isinstance(d, dict):
-            raise IOError(f"Path points to a file : {path}")
+            raise OSError(f"Path points to a file : {path}")
 
         return sorted(d.keys())
 

hydra/core/override_parser/overrides_visitor.py

@@ -107,7 +107,8 @@ def visitDictKeyValuePair(
         item = next(children)
         assert isinstance(item, OverrideParser.DictKeyContext)
         pkey = self.visitDictKey(item)
-        assert self.is_matching_terminal(next(children), OverrideLexer.COLON)
+        colon = next(children)
+        assert self.is_matching_terminal(colon, OverrideLexer.COLON)
         value = next(children)
         assert isinstance(value, OverrideParser.ElementContext)
         return pkey, self.visitElement(value)
@@ -212,7 +213,8 @@ def visitFunction(self, ctx: OverrideParser.FunctionContext) -> Any:
         kwargs = {}
         children = ctx.getChildren()
         func_name = next(children).getText()
-        assert self.is_matching_terminal(next(children), OverrideLexer.POPEN)
+        popen = next(children)
+        assert self.is_matching_terminal(popen, OverrideLexer.POPEN)
         in_kwargs = False
         while True:
             cur = next(children)

hydra/core/singleton.py

@@ -10,7 +10,7 @@ class Singleton(type):
 
     def __call__(cls, *args: Any, **kwargs: Any) -> Any:
         if cls not in cls._instances:
-            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+            cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
     def instance(cls: Any, *args: Any, **kwargs: Any) -> Any:

hydra/errors.py

@@ -12,7 +12,7 @@ class CompactHydraException(HydraException):
 
 class OverrideParseException(CompactHydraException):
     def __init__(self, override: str, message: str) -> None:
-        super(OverrideParseException, self).__init__(message)
+        super().__init__(message)
         self.override = override
         self.message = message
 
@@ -36,7 +36,7 @@ def __init__(
         missing_cfg_file: Optional[str] = None,
         options: Optional[Sequence[str]] = None,
     ) -> None:
-        super(MissingConfigException, self).__init__(message)
+        super().__init__(message)
         self.missing_cfg_file = missing_cfg_file
         self.options = options
 

hydra/plugins/completion_plugin.py

@@ -107,7 +107,7 @@ def str_rep(in_key: Any, in_value: Any) -> str:
         if config is None:
             return []
         elif OmegaConf.is_config(config):
-            matches = []
+            matches: List[str] = []
             if word.endswith(".") or word.endswith("="):
                 exact_key = word[0:-1]
                 try:
@@ -127,15 +127,15 @@ def str_rep(in_key: Any, in_value: Any) -> str:
                 else:
                     key_matches = []
 
-                matches.extend([f"{word}{match}" for match in key_matches])
+                matches.extend(f"{word}{match}" for match in key_matches)
             else:
                 last_dot = word.rfind(".")
                 if last_dot != -1:
                     base_key = word[0:last_dot]
                     partial_key = word[last_dot + 1 :]
                     conf_node = OmegaConf.select(config, base_key)
                     key_matches = CompletionPlugin._get_matches(conf_node, partial_key)
-                    matches.extend([f"{base_key}.{match}" for match in key_matches])
+                    matches.extend(f"{base_key}.{match}" for match in key_matches)
                 else:
                     if isinstance(config, DictConfig):
                         for key, value in config.items_ex(resolve=False):

hydra/test_utils/test_utils.py

@@ -221,7 +221,7 @@ def find_parent_dir_containing(
         cur = os.path.relpath(os.path.join(cur, ".."))
         max_up = max_up - 1
     if max_up == 0:
-        raise IOError(f"Could not find {target} in parents of {os.getcwd()}")
+        raise OSError(f"Could not find {target} in parents of {os.getcwd()}")
     return cur
 
 
@@ -333,7 +333,7 @@ def experiment(cfg):
                 modified_env.update(env_override)
         subprocess.check_call(cmd, env=modified_env)
 
-        with open(output_file, "r") as f:
+        with open(output_file) as f:
             file_str = f.read()
             output = str.splitlines(file_str)