Preemptively fail unparsable/unusable Europeana URLs (#3845)

catalog/dags/providers/provider_api_scripts/europeana.py

@@ -98,7 +98,15 @@ def get_record_data(self, data: dict) -> dict | None:
  @raise_if_empty
  def _get_image_url(self, data: dict) -> str | None:
  group = data.get("edmIsShownBy")
- return group[0] if group else None
+ if not group:
+ return None
+ url = group[0]
+ # Some Europeana URLs may have prefixes, or reference Dropbox (which we can't
+ # include in our catalog because we cannot access them directly ourselves).
+ # E.g.: L-APC248-https://www.dropbox.com/s/i1pqizm1joof8y1/Belgium_Diptyque%20_MAR-SGP-CO1.jpg?raw=1
+ if "dropbox.com" in url:
+ return None
+ return url
 
  @raise_if_empty
  def _get_foreign_identifier(self, data: dict) -> str | None:

catalog/tests/dags/providers/provider_api_scripts/test_europeana.py

@@ -9,6 +9,7 @@
 from common.loader import provider_details as prov
 from common.storage.image import ImageStore
 from providers.provider_api_scripts.europeana import (
+ EmptyRequiredFieldException,
  EuropeanaDataIngester,
  EuropeanaRecordBuilder,
 )
@@ -254,6 +255,19 @@ def test_get_foreign_landing_url_without_edmIsShownAt(record_builder):
  )
 
 
+@pytest.mark.parametrize(
+ "data",
+ [
+ {},
+ {"edmIsShownBy": None},
+ {"edmIsShownBy": ["dropbox.com/value"]},
+ ],
+)
+def test_get_image_url_empty(data, record_builder):
+ with pytest.raises(EmptyRequiredFieldException):
+ assert record_builder._get_image_url(data)
+
+
 @pytest.mark.parametrize(
  "item_data, expected",
  [