🐛 [BUG] Fix errors in Aggregator when retrieving unpublished Route St…

…eps (refs #3569)

geotrek/api/v2/views/trekking.py

@@ -124,6 +124,13 @@ class TrekRatingScaleViewSet(api_viewsets.GeotrekViewSet):
     queryset = trekking_models.RatingScale.objects \
         .order_by('pk')  # Required for reliable pagination
 
+    @cache_response_detail()
+    def retrieve(self, request, pk=None, format=None):
+        # Allow to retrieve objects even if not visible in list view
+        elem = get_object_or_404(trekking_models.RatingScale, pk=pk)
+        serializer = api_serializers.TrekRatingScaleSerializer(elem, many=False, context={'request': request})
+        return Response(serializer.data)
+
 
 class TrekRatingViewSet(api_viewsets.GeotrekViewSet):
     filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (
@@ -134,6 +141,13 @@ class TrekRatingViewSet(api_viewsets.GeotrekViewSet):
     queryset = trekking_models.Rating.objects \
         .order_by('order', 'name', 'pk')  # Required for reliable pagination
 
+    @cache_response_detail()
+    def retrieve(self, request, pk=None, format=None):
+        # Allow to retrieve objects even if not visible in list view
+        elem = get_object_or_404(trekking_models.Rating, pk=pk)
+        serializer = api_serializers.TrekRatingSerializer(elem, many=False, context={'request': request})
+        return Response(serializer.data)
+
 
 class NetworkViewSet(api_viewsets.GeotrekViewSet):
     filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.TrekRelatedPortalFilter,)
@@ -189,12 +203,26 @@ class AccessibilityViewSet(api_viewsets.GeotrekViewSet):
     serializer_class = api_serializers.AccessibilitySerializer
     queryset = trekking_models.Accessibility.objects.all()
 
+    @cache_response_detail()
+    def retrieve(self, request, pk=None, format=None):
+        # Allow to retrieve objects even if not visible in list view
+        elem = get_object_or_404(trekking_models.Accessibility, pk=pk)
+        serializer = api_serializers.AccessibilitySerializer(elem, many=False, context={'request': request})
+        return Response(serializer.data)
+
 
 class AccessibilityLevelViewSet(api_viewsets.GeotrekViewSet):
     filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.TrekRelatedPortalFilter,)
     serializer_class = api_serializers.AccessibilityLevelSerializer
     queryset = trekking_models.AccessibilityLevel.objects.all()
 
+    @cache_response_detail()
+    def retrieve(self, request, pk=None, format=None):
+        # Allow to retrieve objects even if not visible in list view
+        elem = get_object_or_404(trekking_models.AccessibilityLevel, pk=pk)
+        serializer = api_serializers.AccessibilityLevelSerializer(elem, many=False, context={'request': request})
+        return Response(serializer.data)
+
 
 class RouteViewSet(api_viewsets.GeotrekViewSet):
     filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.TrekRelatedPortalFilter,)

geotrek/common/parsers.py

@@ -1323,6 +1323,10 @@ def __init__(self, all_datas=None, create_categories=None, provider=None, mappin
             f.name: f.name
             for f in self.model._meta.many_to_many
         }
+        self.updated_after = None
+        available_fields = [field.name for field in self.model._meta.get_fields()]
+        if not self.all_datas and self.model.objects.filter(provider__exact=self.provider).exists() and 'date_update' in available_fields:
+            self.updated_after = self.model.objects.filter(provider__exact=self.provider).latest('date_update').date_update.strftime('%Y-%m-%d')
         # Replace automatics fields and m2m_fields generated above
         for key, value in self.replace_fields.items():
             self.fields[key] = value
@@ -1455,15 +1459,10 @@ def next_row(self):
         :returns row
         """
         portals = self.portals_filter
-        updated_after = None
-
-        available_fields = [field.name for field in self.model._meta.get_fields()]
-        if not self.all_datas and self.model.objects.filter(provider__exact=self.provider).exists() and 'date_update' in available_fields:
-            updated_after = self.model.objects.filter(provider__exact=self.provider).latest('date_update').date_update.strftime('%Y-%m-%d')
         params = {
             'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]),
             'portals': ','.join(portals) if portals else '',
-            'updated_after': updated_after
+            'updated_after': self.updated_after
         }
         self.params_used = params
         response = self.request_or_retry(self.next_url, params=params)

geotrek/trekking/parsers.py

@@ -22,7 +22,7 @@
 from geotrek.common.models import Label, Theme
 from geotrek.common.parsers import (ApidaeBaseParser, AttachmentParserMixin,
                                     GeotrekParser, GlobalImportError, Parser,
-                                    RowImportError, ShapeParser)
+                                    RowImportError, ShapeParser, DownloadImportError)
 from geotrek.core.models import Path, Topology
 from geotrek.trekking.models import (POI, Accessibility, DifficultyLevel,
                                      OrderedTrekChild, Service, Trek,
@@ -195,19 +195,54 @@ def filter_points_reference(self, src, val):
             geom = GEOSGeometry(json.dumps(val))
             return geom.transform(settings.SRID, clone=True)
 
+    def fetch_missing_categories_for_tour_steps(self, step):
+        # For all categories, search missing values in mapping
+        for category, route in self.url_categories.items():
+            category_mapping = self.field_options.get(category, {}).get('mapping', {})
+            category_values = step.get(category)
+            if not isinstance(category_values, list):
+                category_values = [category_values]
+            for value in category_values:
+                # If category PK is not found in mapping, fetch it from provider
+                if value not in list(category_mapping.keys()):
+                    if self.categories_keys_api_v2.get(category):
+                        try:
+                            result = self.request_or_retry(f"{self.url}/api/v2/{route}/{int(value)}").json()
+                        except DownloadImportError:
+                            self.add_warning(f"Could not download {category} with id {value} from {self.provider}" +
+                                             f" for Tour step {step.get('uuid')}")
+                            continue
+                        # Update mapping like we usually do
+                        label = result[self.categories_keys_api_v2[category]]
+                        if isinstance(result[self.categories_keys_api_v2[category]], dict):
+                            if label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE]:
+                                self.field_options[category]["mapping"][value] = self.replace_mapping(label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE], route)
+                        else:
+                            if label:
+                                self.field_options[category]["mapping"][value] = self.replace_mapping(label, category)
+
     def end(self):
         """Add children after all treks imported are created in database."""
         self.next_url = f"{self.url}/api/v2/tour"
+        portals = self.portals_filter
         try:
             params = {
                 'in_bbox': ','.join([str(coord) for coord in self.bbox.extent]),
-                'fields': 'steps,id'
+                'fields': 'steps,id,uuid,published',
+                'portals': ','.join(portals) if portals else '',
+                'updated_after': self.updated_after
             }
             response = self.request_or_retry(f"{self.next_url}", params=params)
             results = response.json()['results']
+            steps_to_download = 0
             final_children = {}
             for result in results:
-                final_children[result['uuid']] = [step['id'] for step in result['steps']]
+                final_children[result['uuid']] = []
+                for step in result['steps']:
+                    final_children[result['uuid']].append(step['id'])
+                    if not any(step['published'].values()):
+                        steps_to_download += 1
+            self.nb = steps_to_download
 
             for key, value in final_children.items():
                 if value:
@@ -219,6 +254,9 @@ def end(self):
                     for child_id in value:
                         response = self.request_or_retry(f"{self.url}/api/v2/trek/{child_id}")
                         child_trek = response.json()
+                        # The Tour step might be linked to categories that are not published,
+                        # we have to retrieve the missing ones first
+                        self.fetch_missing_categories_for_tour_steps(child_trek)
                         self.parse_row(child_trek)
                         trek_child_instance = self.obj
                         OrderedTrekChild.objects.update_or_create(parent=trek_parent_instance[0],

geotrek/trekking/tests/data/geotrek_parser_v2/trek_children.json

@@ -7,10 +7,20 @@
       "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1",
       "steps": [
         {
-          "id": 10439
+          "id": 10439,
+          "published": {
+            "fr": true,
+            "en": false,
+            "it": false
+          }
         },
         {
-          "id": 10442
+          "id": 10442,
+          "published": {
+            "fr": false,
+            "en": false,
+            "it": false
+          }
         }
       ]
     },

geotrek/trekking/tests/data/geotrek_parser_v2/trek_children_do_not_exist.json

@@ -7,10 +7,20 @@
       "uuid": "9e70b294-1134-4c50-9c56-d722720cacf1",
       "steps": [
         {
-          "id": 1234
+          "id": 1234,
+          "published": {
+            "fr": true,
+            "en": false,
+            "it": false
+          }
         },
         {
-          "id": 1235
+          "id": 1235,
+          "published": {
+            "fr": false,
+            "en": false,
+            "it": false
+          }
         }
       ]
     },
@@ -34,7 +44,12 @@
       "uuid": "b2aea666-5e6e-4daa-a750-7d2ee52d3fe1",
       "steps": [
         {
-          "id": 457
+          "id": 457,
+          "published": {
+            "fr": false,
+            "en": false,
+            "it": false
+          }
         }
       ]
     }

geotrek/trekking/tests/data/geotrek_parser_v2/trek_network.json

@@ -7,7 +7,7 @@
       "id": 2,
       "label": {
         "fr": "PR",
-        "en": null,
+        "en": "PR",
         "es": null,
         "it": null
       },
@@ -17,7 +17,7 @@
       "id": 4,
       "label": {
         "fr": "VTT",
-        "en": null,
+        "en": "Bike",
         "es": null,
         "it": null
       },

geotrek/trekking/tests/data/geotrek_parser_v2/trek_published_step.json

@@ -174,7 +174,7 @@
   },
   "points_reference": null,
   "portal": [],
-  "practice": 4,
+  "practice": 1,
   "ratings": [],
   "ratings_description": {
     "fr": "",
@@ -197,7 +197,7 @@
   },
   "reservation_system": null,
   "reservation_id": "",
-  "route": 3,
+  "route": 1,
   "second_external_id": null,
   "source": [],
   "structure": 3,

geotrek/trekking/tests/data/geotrek_parser_v2/trek_route.json

@@ -8,7 +8,7 @@
       "pictogram": "https://foo.fr/media/upload/route-loop.svg",
       "route": {
         "fr": "Boucle",
-        "en": null,
+        "en": "Loop",
         "es": null,
         "it": null
       }
@@ -18,7 +18,7 @@
       "pictogram": "https://foo.fr/media/upload/route-return.svg",
       "route": {
         "fr": "Aller-retour",
-        "en": null,
+        "en": "Return trip",
         "es": null,
         "it": null
       }
@@ -28,7 +28,7 @@
       "pictogram": "https://foo.fr/media/upload/route-cross.svg",
       "route": {
         "fr": "Traversée",
-        "en": null,
+        "en": "Crossing",
         "es": null,
         "it": null
       }

geotrek/trekking/tests/data/geotrek_parser_v2/trek_unpublished_step.json

@@ -5,9 +5,7 @@
       "en": "",
       "it": ""
     },
-    "accessibilities": [
-      1
-    ],
+    "accessibilities": [],
     "accessibility_advice": {
       "fr": "",
       "en": "",
@@ -233,8 +231,8 @@
       "it": ""
     },
     "published": {
-      "fr": true,
-      "en": true,
+      "fr": false,
+      "en": false,
       "it": false
     },
     "reservation_system": null,
@@ -246,8 +244,8 @@
     ],
     "structure": 1,
     "themes": [
-      8,
-      4
+      1,
+      2
     ],
     "update_datetime": "2022-05-16T12:10:59.927409Z",
     "url": "https://foo.fr/api/v2/trek/2/",

geotrek/trekking/tests/test_parsers.py

@@ -16,6 +16,7 @@
 from django.test import TestCase, SimpleTestCase
 from django.test.utils import override_settings
 
+from geotrek.common.parsers import DownloadImportError
 from geotrek.common.utils import testdata
 from geotrek.common.utils.file_infos import get_encoding_file
 from geotrek.common.models import Theme, FileType, Attachment, Label
@@ -667,6 +668,17 @@ def test_children_do_not_exist(self, mocked_head, mocked_get):
     @mock.patch('requests.get')
     @mock.patch('requests.head')
     def test_wrong_children_error(self, mocked_head, mocked_get):
+
+        def mock_json_with_404():
+            filename = os.path.join('geotrek', self.mock_json_order[self.mock_time][0], 'tests', 'data',
+                                    'geotrek_parser_v2',
+                                    self.mock_json_order[self.mock_time][1])
+            self.mock_time += 1
+            if "trek_not_found" in filename:
+                raise DownloadImportError("404 Trek does not exist")
+            with open(filename, 'r') as f:
+                return json.load(f)
+
         self.mock_time = 0
         self.mock_json_order = [('trekking', 'structure.json'),
                                 ('trekking', 'trek_difficulty.json'),
@@ -685,14 +697,14 @@ def test_wrong_children_error(self, mocked_head, mocked_get):
 
         # Mock GET
         mocked_get.return_value.status_code = 200
-        mocked_get.return_value.json = self.mock_json
+        mocked_get.return_value.json = mock_json_with_404
         mocked_get.return_value.content = b''
         mocked_head.return_value.status_code = 200
         output = StringIO()
 
         call_command('import', 'geotrek.trekking.tests.test_parsers.TestGeotrekTrekParser', verbosity=2,
                      stdout=output)
-        self.assertIn("An error occured in children generation : ValueImportError", output.getvalue())
+        self.assertIn("An error occured in children generation : DownloadImportError", output.getvalue())
 
     @mock.patch('requests.get')
     @mock.patch('requests.head')