add ! aurora import

.pre-commit-config.yaml

@@ -3,20 +3,20 @@ repos:
  rev: 5.13.2
  hooks:
  - id: isort
- stages: [pre-commit]
+ stages: [commit]
  - repo: https://github.com/ambv/black
  rev: 24.4.2
  hooks:
  - id: black
  args: [--config=pyproject.toml]
  exclude: "migrations|snapshots"
- stages: [pre-commit]
+ stages: [commit]
  - repo: https://github.com/PyCQA/flake8
  rev: 7.1.0
  hooks:
  - id: flake8
  args: [--config=.flake8]
- stages: [ pre-commit ]
+ stages: [ commit ]
  - repo: https://github.com/PyCQA/bandit
  rev: '1.7.9' # Update me!
  hooks:
@@ -33,20 +33,20 @@ repos:
  - id: check-missed-migrations
  args:
  - src
- stages: [ pre-commit ]
+ stages: [ commit ]
  additional_dependencies: [ setuptools ]
 
  - id: check-untracked
  args:
  - src
  - tests
- stages: [ pre-push ]
+ stages: [ push ]
 
  - id: check-forbidden
  args:
  - -p
  - /\.showbrowser\(/
  - -p
  - /print\(111/
- stages: [ pre-commit ]
+ stages: [ commit ]
  additional_dependencies: [ setuptools ]

src/country_workspace/config/__init__.py

@@ -59,6 +59,8 @@ class Group(Enum):
  False,
  "Extra authentications backends enabled to add. Es. `country_workspace.security.backends.AnyUserAuthBackend`",
  ),
+ "AURORA_API_TOKEN": (str, "", "", True, "Aurora API token"),
+ "AURORA_API_URL": (str, "", "", True, "Aurora API url"),
  "CACHE_URL": (str, "", "redis://localhost:6379/0", True, setting("cache-url")),
  "CELERY_BROKER_URL": (
  str,
@@ -89,7 +91,7 @@ class Group(Enum):
  celery_doc("#broker-transport-options"),
  ),
  "CSRF_COOKIE_SECURE": (bool, True, False, True, setting("csrf-cookie-secure")),
- "CSRF_TRUSTED_ORIGINS": (list, "localhost", "", True, ""),
+ "CSRF_TRUSTED_ORIGINS": (list, ["http://localhost"], "", True, ""),
  "DATABASE_URL": (
  str,
  SmartEnv.NOTSET,

src/country_workspace/config/fragments/aurora.py

@@ -0,0 +1,3 @@
+from ..settings import env # type: ignore[attr-defined]
+
+AURORA_API_TOKEN = env("AURORA_API_TOKEN")

src/country_workspace/config/fragments/constance.py

@@ -1,4 +1,4 @@
-from ..settings import NEW_USER_DEFAULT_GROUP
+from ..settings import NEW_USER_DEFAULT_GROUP, env
 
 CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend"
 
@@ -29,4 +29,5 @@
  "group_select",
  ),
  "HOPE_API_URL": ("https://hope.unicef.org/api/rest/", "", str),
+ "AURORA_API_URL": (env("AURORA_API_URL"), "", str),
 }

src/country_workspace/config/settings.py

@@ -46,6 +46,7 @@
  "country_workspace.security",
  "country_workspace.apps.Config",
  "country_workspace.workspaces.apps.Config",
+ "country_workspace.contrib.aurora.apps.Config",
  "country_workspace.versioning",
 )
 
@@ -211,6 +212,7 @@
 EMAIL_USE_SSL = env("EMAIL_USE_SSL", default=False)
 
 from .fragments.app import * # noqa
+from .fragments.aurora import * # noqa
 from .fragments.celery import * # noqa
 from .fragments.constance import * # noqa
 from .fragments.csp import * # noqa

src/country_workspace/contrib/aurora/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class Config(AppConfig):
+ name = __name__.rpartition(".")[0]
+ verbose_name = "Country Workspace | Aurora"

src/country_workspace/contrib/aurora/client.py

@@ -0,0 +1,51 @@
+from typing import Any, Generator
+from urllib.parse import urljoin
+
+from django.conf import settings
+
+import requests
+from constance import config
+
+
+class AuroraClient:
+
+ def __init__(self, token: str | None = None) -> None:
+ self.token = token or settings.AURORA_API_TOKEN
+ # temporarily for development purposes
+ self.use_mock = True
+
+ def _get_url(self, path: str) -> str:
+ url = urljoin(config.AURORA_API_URL, path)
+ if not url.endswith("/"):
+ url = url + "/"
+ return url
+
+ def _load_mock_data(self, path: str) -> dict:
+ import json
+ from pathlib import Path
+
+ file_path = (
+ Path(__file__).resolve().parent.parent.parent.parent.parent / "tests" / "extras" / "aurora_record.json"
+ )
+ if file_path.exists():
+ with file_path.open() as f:
+ return json.load(f)
+ raise FileNotFoundError(f"Mock data for {path} not found")
+
+ def get(self, path: str) -> Generator[dict[str, Any], None, None]:
+ if self.use_mock:
+ data = self._load_mock_data(path)
+ else:
+ url = self._get_url(path)
+ ret = requests.get(url, headers={"Authorization": f"Token {self.token}"}, timeout=10)
+ if ret.status_code != 200:
+ raise Exception(f"Error {ret.status_code} fetching {url}")
+ data = ret.json()
+
+ for record in data["results"]:
+ yield record
+
+ if not self.use_mock and "next" in data:
+ url = data["next"]
+ else:
+ url = None

src/country_workspace/workspaces/admin/forms.py

@@ -30,3 +30,7 @@ class ImportFileForm(forms.Form):
  first_line = forms.IntegerField(required=True, initial=0, help_text="First line to process")
  fail_if_alien = forms.BooleanField(required=False)
  file = forms.FileField(validators=[ValidatableFileValidator()])
+
+
+class ImportAuroraForm(ImportFileForm):
+ pk_column_name = description_column_name = first_line = file = None

src/country_workspace/workspaces/admin/program.py

@@ -17,11 +17,12 @@
 
 from country_workspace.state import state
 
+from ...contrib.aurora.client import AuroraClient
 from ...models import Batch
 from ...sync.office import sync_programs
 from ..models import CountryProgram
 from ..options import WorkspaceModelAdmin
-from .forms import ImportFileForm
+from .forms import ImportAuroraForm, ImportFileForm
 
 
 class SelectColumnsForm(forms.Form):
@@ -206,3 +207,52 @@ def import_rdi(self, request: HttpRequest, pk: str) -> "HttpResponse":
  form = ImportFileForm()
  context["form"] = form
  return render(request, "workspace/program/import_rdi.html", context)
+
+ @button(label=_("Import from Aurora"))
+ def import_aurora(self, request: HttpRequest, pk: str) -> "HttpResponse":
+ context = self.get_common_context(request, pk, title="Import from Aurora")
+ program: "CountryProgram" = context["original"]
+ context["selected_program"] = context["original"]
+ client = AuroraClient()
+ if request.method == "POST":
+ form = ImportAuroraForm(request.POST)
+ if form.is_valid():
+ total_hh = total_ind = 0
+ with atomic():
+ batch_name = form.cleaned_data["batch_name"]
+ batch, __ = Batch.objects.get_or_create(
+ name=batch_name or ("Batch %s" % timezone.now()),
+ program=program,
+ country_office=program.country_office,
+ imported_by=request.user,
+ )
+ total_hh = total_ind = 0
+
+ for i, record in enumerate(client.get("record")):
+ for field, f_value in record["fields"].items():
+ try:
+ if field == "household":
+ hh = program.households.create(
+ batch=batch, flex_fields={clean_field_name(k): v for k, v in f_value[0].items()}
+ )
+ total_hh += 1
+ elif field == "individuals":
+ for value in f_value:
+ program.individuals.create(
+ batch=batch,
+ household_id=hh.pk,
+ flex_fields={clean_field_name(k): v for k, v in value.items()},
+ )
+ total_ind += 1
+ except Exception as e:
+ raise Exception("Error processing record %s: %s" % (i, e))
+
+ hh_msg = ngettext("%(c)d Household", "%(c)d Households", total_hh) % {"c": total_hh}
+ ind_msg = ngettext("%(c)d Individual", "%(c)d Individuals", total_ind) % {"c": total_ind}
+ self.message_user(request, _("Imported {0} and {1}").format(hh_msg, ind_msg))
+ context["form"] = form
+
+ else:
+ form = ImportAuroraForm()
+ context["form"] = form
+ return render(request, "workspace/program/import_rdi.html", context)

tests/workspace/test_ws_import.py

@@ -25,6 +25,9 @@ def office():
 def program(request, office, force_migrated_records):
  from testutils.factories import CountryProgramFactory
 
+ DataChecker.objects.create(name=HOUSEHOLD_CHECKER_NAME)
+ DataChecker.objects.create(name=INDIVIDUAL_CHECKER_NAME)
+
  return CountryProgramFactory(
  country_office=office,
  household_checker=DataChecker.objects.get(name=HOUSEHOLD_CHECKER_NAME),