fix: use mirrors.json urls when cloning dependencies (#551)

Refactor config.url -> config.remote_url. This flag is set when running taf repo clone command from CLI.

The current system with taf repo clone uses the config.remote_url to clone the top level authentication repository.
However, for other repositories, we need to try cloning from all the urls in mirrors.json, not just the url that's passed in by the CLI.
To resolve, add config.clone_urls attribute that's set when repos are instantiated using repositoriesdb.

taf/tests/test_updater/update_utils.py

@@ -126,7 +126,7 @@ def clone_repositories(
  shutil.rmtree(client_test_root)
  config = UpdateConfig(
  operation=OperationType.CLONE,
- url=str(origin_auth_repo.path),
+ remote_url=str(origin_auth_repo.path),
  update_from_filesystem=True,
  path=None,
  library_dir=str(clients_dir),
@@ -235,7 +235,7 @@ def update_and_check_commit_shas(
 
  config = UpdateConfig(
  operation=operation,
- url=str(origin_auth_repo.path),
+ remote_url=str(origin_auth_repo.path),
  update_from_filesystem=True,
  path=str(clients_auth_repo_path) if auth_repo_name_exists else None,
  library_dir=str(clients_dir),
@@ -299,7 +299,7 @@ def update_invalid_repos_and_check_if_repos_exist(
 
  config = UpdateConfig(
  operation=operation,
- url=str(origin_auth_repo.path),
+ remote_url=str(origin_auth_repo.path),
  update_from_filesystem=True,
  path=str(clients_auth_repo_path) if auth_repo_name_exists else None,
  library_dir=str(clients_dir),

taf/tools/repo/__init__.py

@@ -154,7 +154,7 @@ def clone(path, url, library_dir, from_fs, expected_repo_type, scripts_root_dir,
 
  config = UpdateConfig(
  operation=OperationType.CLONE,
- url=url,
+ remote_url=url,
  path=path,
  library_dir=library_dir,
  update_from_filesystem=from_fs,

taf/updater/handlers.py

@@ -76,7 +76,7 @@ def metadata_dir(self):
  def targets_dir(self):
  return str(self.validation_auth_repo.path / "targets")
 
- def __init__(self, auth_url, repository_directory, repository_name):
+ def __init__(self, auth_urls, repository_directory, repository_name):
  """
  Args:
  auth_url: repository url of the git repository which we want to clone.
@@ -91,7 +91,7 @@ def __init__(self, auth_url, repository_directory, repository_name):
 
  validation_path = settings.validation_repo_path.get(repository_name)
 
- self.set_validation_repo(validation_path, auth_url)
+ self.set_validation_repo(validation_path, auth_urls)
 
  self._init_commits()
 
@@ -187,11 +187,11 @@ def wrapper(self):
 
  return wrapper
 
- def set_validation_repo(self, path, url):
+ def set_validation_repo(self, path, urls):
  """
  Used outside of GitUpdater to access validation auth repo.
  """
- self.validation_auth_repo = AuthenticationRepository(path=path, urls=[url])
+ self.validation_auth_repo = AuthenticationRepository(path=path, urls=urls)
 
  def cleanup(self):
  """

taf/updater/updater.py

@@ -139,8 +139,9 @@ def _reset_repository(repo, commits_data):
 @define
 class UpdateConfig:
  operation: OperationType = field(converter=OperationType)
- url: str = field(
- metadata={"docs": "URL of the remote authentication repository"}, default=None
+ remote_url: str = field(
+ metadata={"docs": "Remote URL of the remote authentication repository"},
+ default=None,
  )
  path: Path = field(
  default=None,
@@ -203,6 +204,10 @@ class UpdateConfig:
  "docs": "Whether to checkout last validated commits after update. Optional."
  },
  )
+ clone_urls: list = field(
+ default=None,
+ metadata={"docs": "List of URLs to clone repositories from. Optional."},
+ )
  excluded_target_globs: list = field(
  default=None,
  metadata={
@@ -281,8 +286,10 @@ def clone_repository(config: UpdateConfig):
  """
  settings.strict = config.strict
 
- if config.url is None:
- raise UpdateFailedError("URL has to be specified when cloning repositories")
+ if config.remote_url is None:
+ raise UpdateFailedError(
+ "Remote URL has to be specified when cloning repositories"
+ )
 
  if config.path and is_non_empty_directory(config.path):
  raise UpdateFailedError(
@@ -329,9 +336,9 @@ def update_repository(config: UpdateConfig):
 
  taf_logger.info(f"Updating repository {auth_repo.name}")
 
- if config.url is None:
- config.url = auth_repo.get_remote_url()
- if config.url is None:
+ if config.remote_url is None:
+ config.remote_url = auth_repo.get_remote_url()
+ if config.remote_url is None:
  raise UpdateFailedError("URL cannot be determined. Please specify it")
 
  if auth_repo.is_bare_repository:
@@ -419,9 +426,9 @@ def _process_repo_update(
  if visited is None:
  visited = []
  # if there is a recursive dependency
- if update_config.url in visited:
+ if update_config.remote_url in visited:
  return
- visited.append(update_config.url)
+ visited.append(update_config.remote_url)
  # at the moment, we assume that the initial commit is valid and that it contains at least root.json
  update_status = update_output.event
  auth_repo = update_output.users_auth_repo
@@ -481,7 +488,8 @@ def _process_repo_update(
  update_status = Event.FAILED
  repo = output.users_auth_repo
  child_config = copy.copy(update_config)
- child_config.url = repo.urls[0]
+ child_config.remote_url = repo.urls[0]
+ child_config.clone_urls = repo.urls
  child_config.out_of_band_authentication = (
  repo.out_of_band_authentication
  )
@@ -582,7 +590,8 @@ def _update_child_repo(updater_pipeline):
  child_config.operation = (
  OperationType.UPDATE if repo.is_git_repository else OperationType.CLONE
  )
- child_config.url = repo.urls[0]
+ child_config.remote_url = repo.urls[0]
+ child_config.clone_urls = repo.urls
  child_config.out_of_band_authentication = repo.out_of_band_authentication
  child_config.path = repo.path
  pipeline = AuthenticationRepositoryUpdatePipeline(child_config)
@@ -638,7 +647,7 @@ def validate_repository(
  try:
  config = UpdateConfig(
  operation=OperationType.UPDATE,
- url=str(auth_path),
+ remote_url=str(auth_path),
  path=auth_path,
  library_dir=library_dir,
  validate_from_commit=validate_from_commit,

taf/updater/updater_pipeline.py

@@ -347,7 +347,7 @@ def __init__(
  ),
  )
  self.operation = update_config.operation
- self.url = update_config.url
+ self.urls = update_config.clone_urls or [update_config.remote_url]
  self.library_dir = update_config.library_dir
  self.auth_path = update_config.path
  self.update_from_filesystem = update_config.update_from_filesystem
@@ -410,7 +410,7 @@ def set_users_auth_repo(self):
  self.state.existing_repo = False
  if self.auth_path:
  self.state.users_auth_repo = AuthenticationRepository(
- path=self.auth_path, urls=[self.url]
+ path=self.auth_path, urls=self.urls
  )
  self.state.auth_repo_name = self.state.users_auth_repo.name
  if self.state.users_auth_repo.is_git_repository_root:
@@ -468,7 +468,7 @@ def check_if_local_repositories_clean(self):
  if not self.state.existing_repo:
  return UpdateStatus.SUCCESS
 
- auth_repo = AuthenticationRepository(path=self.auth_path, urls=[self.url])
+ auth_repo = AuthenticationRepository(path=self.auth_path, urls=self.urls)
  taf_logger.info(
  f"{auth_repo.name}: Checking if local repositories are clean..."
  )
@@ -594,7 +594,7 @@ def clone_auth_to_temp(self):
 
  path = Path(self.state.temp_root.temp_dir, "auth_repo").absolute()
  self.state.validation_auth_repo = AuthenticationRepository(
- path=path, urls=[self.url], alias="Validation repository"
+ path=path, urls=self.urls, alias="Validation repository"
  )
  self.state.validation_auth_repo.clone(bare=True)
  self.state.validation_auth_repo.fetch(fetch_all=True)
@@ -760,7 +760,7 @@ def run_tuf_updater(self):
  try:
 
  self.state.update_handler = GitUpdater(
- self.url, self.library_dir, self.state.validation_auth_repo.name
+ self.urls, self.library_dir, self.state.validation_auth_repo.name
  )
  last_validated_remote_commit, error = _run_tuf_updater(
  self.state.update_handler, self.state.auth_repo_name
@@ -791,7 +791,7 @@ def run_tuf_updater(self):
  self.state.users_auth_repo = AuthenticationRepository(
  library_dir=self.library_dir,
  name=self.state.auth_repo_name,
- urls=[self.url],
+ urls=self.urls,
  )
 
  self._validate_operation_type()
@@ -854,7 +854,7 @@ def run_tuf_updater(self):
  self.state.users_auth_repo = AuthenticationRepository(
  self.library_dir,
  self.state.auth_repo_name,
- urls=[self.url],
+ urls=self.urls,
  )
 
  def _validate_operation_type(self):