diff --git a/internal/server/v2/observation/contained_in.go b/internal/server/v2/observation/contained_in.go index 901856d49..cf2e2b4db 100644 --- a/internal/server/v2/observation/contained_in.go +++ b/internal/server/v2/observation/contained_in.go @@ -186,6 +186,11 @@ func FetchContainedIn( var err error if store.BtGroup != nil { readCollectionCache := false + childPlaces, err = FetchChildPlaces( + ctx, store, metadata, httpClient, remoteMixer, ancestor, childType) + if err != nil { + return nil, err + } if queryDate != "" { result = &pbv2.ObservationResponse{ ByVariable: map[string]*pbv2.VariableObservation{}, @@ -210,6 +215,10 @@ func FetchContainedIn( obsByEntity := result.ByVariable[variable].ByEntity data, ok := btData[variable] if !ok || data == nil { + // If data is missing, attach empty child places. + for _, place := range childPlaces { + obsByEntity[place] = &pbv2.EntityObservation{} + } continue } cohorts := data.SourceCohorts @@ -257,11 +266,6 @@ func FetchContainedIn( } } if !readCollectionCache { - childPlaces, err = FetchChildPlaces( - ctx, store, metadata, httpClient, remoteMixer, ancestor, childType) - if err != nil { - return nil, err - } totalSeries := len(variables) * len(childPlaces) if totalSeries > MaxSeries { return nil, status.Errorf( diff --git a/internal/server/v2/observation/golden/calculation/contained_in_2015.json b/internal/server/v2/observation/golden/calculation/contained_in_2015.json new file mode 100644 index 000000000..e55705e6f --- /dev/null +++ b/internal/server/v2/observation/golden/calculation/contained_in_2015.json @@ -0,0 +1,225 @@ +{ + "by_variable": { + "Count_Person_Female": { + "by_entity": { + "wikidataId/Q159477": { + "ordered_facets": [ + { + "observations": [ + { + "date": "1975", + "value": 6170 + }, + { + "date": "1980", + "value": 13856 + }, + { + "date": "1985", + "value": 21711 + }, + { + "date": "1995", + "value": 48659 + }, + { + "date": "2005", + "value": 75313 + } + ], + "obs_count": 5, + "earliest_date": "1975", + "latest_date": "2005" + } + ] + }, + "wikidataId/Q170024": { + "ordered_facets": [ + { + "observations": [ + { + "date": "1975", + "value": 17307 + }, + { + "date": "1980", + "value": 27393 + }, + { + "date": "1985", + "value": 40165 + }, + { + "date": "1995", + "value": 59342 + }, + { + "date": "2005", + "value": 80621 + } + ], + "obs_count": 5, + "earliest_date": "1975", + "latest_date": "2005" + } + ] + }, + "wikidataId/Q175021": { + "ordered_facets": [ + { + "observations": [ + { + "date": "1975", + "value": 2645 + }, + { + "date": "1980", + "value": 4466 + }, + { + "date": "1985", + "value": 7626 + }, + { + "date": "1995", + "value": 14414 + }, + { + "date": "2005", + "value": 19075 + } + ], + "obs_count": 5, + "earliest_date": "1975", + "latest_date": "2005" + } + ] + }, + "wikidataId/Q187712": { + "ordered_facets": [ + { + "observations": [ + { + "date": "1975", + "value": 56754 + }, + { + "date": "1980", + "value": 120160 + }, + { + "date": "1985", + "value": 185783 + }, + { + "date": "1995", + "value": 291719 + }, + { + "date": "2005", + "value": 472665 + } + ], + "obs_count": 5, + "earliest_date": "1975", + "latest_date": "2005" + } + ] + }, + "wikidataId/Q188810": { + "ordered_facets": [ + { + "observations": [ + { + "date": "1975", + "value": 27535 + }, + { + "date": "1980", + "value": 56532 + }, + { + "date": "1985", + "value": 86466 + }, + { + "date": "1995", + "value": 148478 + }, + { + "date": "2005", + "value": 273339 + } + ], + "obs_count": 5, + "earliest_date": "1975", + "latest_date": "2005" + } + ] + }, + "wikidataId/Q4091": { + "ordered_facets": [ + { + "observations": [ + { + "date": "1975", + "value": 6683 + }, + { + "date": "1980", + "value": 11393 + }, + { + "date": "1985", + "value": 18343 + }, + { + "date": "1995", + "value": 30414 + }, + { + "date": "2005", + "value": 47114 + } + ], + "obs_count": 5, + "earliest_date": "1975", + "latest_date": "2005" + } + ] + }, + "wikidataId/Q613": { + "ordered_facets": [ + { + "observations": [ + { + "date": "1975", + "value": 54366 + }, + { + "date": "1980", + "value": 88587 + }, + { + "date": "1985", + "value": 123609 + }, + { + "date": "1995", + "value": 211211 + }, + { + "date": "2005", + "value": 332148 + } + ], + "obs_count": 5, + "earliest_date": "1975", + "latest_date": "2005" + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/internal/server/v2/observation/golden/calculation_test.go b/internal/server/v2/observation/golden/calculation_test.go index 53a5bfb4a..a0fe7e31e 100644 --- a/internal/server/v2/observation/golden/calculation_test.go +++ b/internal/server/v2/observation/golden/calculation_test.go @@ -141,3 +141,70 @@ func TestCalculation(t *testing.T) { t.Errorf("TestDriver() = %s", err) } } + +func TestCalculationForObsCollection(t *testing.T) { + t.Parallel() + ctx := context.Background() + + _, filename, _, _ := runtime.Caller(0) + goldenPath := path.Join(path.Dir(filename), "calculation") + + testSuite := func(mixer pbs.MixerClient, latencyTest bool) { + for _, c := range []struct { + desc string + variables []string + entityExpression string + date string + filter *pbv2.FacetFilter + goldenFile string + }{ + { + "contained in 2015", + []string{ + "Count_Person_Female", + }, + "country/ARE<-containedInPlace+{typeOf:AdministrativeArea1}", + "2015", + nil, + "contained_in_2015.json", + }, + } { + goldenFile := c.goldenFile + resp, err := mixer.V2Observation(ctx, &pbv2.ObservationRequest{ + Select: []string{"variable", "entity", "date", "value"}, + Variable: &pbv2.DcidOrExpression{Dcids: c.variables}, + Entity: &pbv2.DcidOrExpression{Expression: c.entityExpression}, + Date: c.date, + Filter: c.filter, + }) + if err != nil { + t.Errorf("could not run V2Observation (direct): %s", err) + continue + } + if latencyTest { + continue + } + if test.GenerateGolden { + test.UpdateGolden(resp, goldenPath, goldenFile) + continue + } + var expected pbv2.ObservationResponse + if err = test.ReadJSON(goldenPath, goldenFile, &expected); err != nil { + t.Errorf("Can not Unmarshal golden file: %s", err) + continue + } + if diff := cmp.Diff(resp, &expected, protocmp.Transform()); diff != "" { + t.Errorf("%s: got diff: %s", c.desc, diff) + continue + } + } + } + if err := test.TestDriver( + "TestCalculationForObsCollection", + &test.TestOption{UseSQLite: true, CacheSVFormula: true}, + testSuite, + ); err != nil { + t.Errorf("TestDriver() = %s", err) + } + +} diff --git a/internal/server/v2/observation/golden/contained_in_2015/US_State.json b/internal/server/v2/observation/golden/contained_in_2015/US_State.json index 2cbcb9e11..94406da97 100644 --- a/internal/server/v2/observation/golden/contained_in_2015/US_State.json +++ b/internal/server/v2/observation/golden/contained_in_2015/US_State.json @@ -2,6 +2,10 @@ "by_variable": { "test_var_1": { "by_entity": { + "geoId/01": {}, + "geoId/02": {}, + "geoId/04": {}, + "geoId/05": {}, "geoId/06": { "ordered_facets": [ { @@ -17,7 +21,54 @@ "latest_date": "2015" } ] - } + }, + "geoId/08": {}, + "geoId/09": {}, + "geoId/10": {}, + "geoId/11": {}, + "geoId/12": {}, + "geoId/13": {}, + "geoId/15": {}, + "geoId/16": {}, + "geoId/17": {}, + "geoId/18": {}, + "geoId/19": {}, + "geoId/20": {}, + "geoId/21": {}, + "geoId/22": {}, + "geoId/23": {}, + "geoId/24": {}, + "geoId/25": {}, + "geoId/26": {}, + "geoId/27": {}, + "geoId/28": {}, + "geoId/29": {}, + "geoId/30": {}, + "geoId/31": {}, + "geoId/32": {}, + "geoId/33": {}, + "geoId/34": {}, + "geoId/35": {}, + "geoId/36": {}, + "geoId/37": {}, + "geoId/38": {}, + "geoId/39": {}, + "geoId/40": {}, + "geoId/41": {}, + "geoId/42": {}, + "geoId/44": {}, + "geoId/45": {}, + "geoId/46": {}, + "geoId/47": {}, + "geoId/48": {}, + "geoId/49": {}, + "geoId/50": {}, + "geoId/51": {}, + "geoId/53": {}, + "geoId/54": {}, + "geoId/55": {}, + "geoId/56": {}, + "geoId/72": {} } } }, diff --git a/internal/server/v2/observation/golden/contained_in_latest/CA_County.json b/internal/server/v2/observation/golden/contained_in_latest/CA_County.json index ce8abc783..2310ff070 100644 --- a/internal/server/v2/observation/golden/contained_in_latest/CA_County.json +++ b/internal/server/v2/observation/golden/contained_in_latest/CA_County.json @@ -10455,7 +10455,68 @@ } } }, - "dummy": {} + "dummy": { + "by_entity": { + "geoId/06001": {}, + "geoId/06003": {}, + "geoId/06005": {}, + "geoId/06007": {}, + "geoId/06009": {}, + "geoId/06011": {}, + "geoId/06013": {}, + "geoId/06015": {}, + "geoId/06017": {}, + "geoId/06019": {}, + "geoId/06021": {}, + "geoId/06023": {}, + "geoId/06025": {}, + "geoId/06027": {}, + "geoId/06029": {}, + "geoId/06031": {}, + "geoId/06033": {}, + "geoId/06035": {}, + "geoId/06037": {}, + "geoId/06039": {}, + "geoId/06041": {}, + "geoId/06043": {}, + "geoId/06045": {}, + "geoId/06047": {}, + "geoId/06049": {}, + "geoId/06051": {}, + "geoId/06053": {}, + "geoId/06055": {}, + "geoId/06057": {}, + "geoId/06059": {}, + "geoId/06061": {}, + "geoId/06063": {}, + "geoId/06065": {}, + "geoId/06067": {}, + "geoId/06069": {}, + "geoId/06071": {}, + "geoId/06073": {}, + "geoId/06075": {}, + "geoId/06077": {}, + "geoId/06079": {}, + "geoId/06081": {}, + "geoId/06083": {}, + "geoId/06085": {}, + "geoId/06087": {}, + "geoId/06089": {}, + "geoId/06091": {}, + "geoId/06093": {}, + "geoId/06095": {}, + "geoId/06097": {}, + "geoId/06099": {}, + "geoId/06101": {}, + "geoId/06103": {}, + "geoId/06105": {}, + "geoId/06107": {}, + "geoId/06109": {}, + "geoId/06111": {}, + "geoId/06113": {}, + "geoId/06115": {} + } + } }, "facets": { "103184633": { diff --git a/internal/server/v2/observation/golden/contained_in_latest/US_State.json b/internal/server/v2/observation/golden/contained_in_latest/US_State.json index ee1169346..8484649ab 100644 --- a/internal/server/v2/observation/golden/contained_in_latest/US_State.json +++ b/internal/server/v2/observation/golden/contained_in_latest/US_State.json @@ -4604,7 +4604,15 @@ "latest_date": "2023" } ] - } + }, + "geoId/49": {}, + "geoId/50": {}, + "geoId/51": {}, + "geoId/53": {}, + "geoId/54": {}, + "geoId/55": {}, + "geoId/56": {}, + "geoId/72": {} } } }, diff --git a/test/datacommons.db b/test/datacommons.db index 4e3efd5ec..846ffeef2 100644 Binary files a/test/datacommons.db and b/test/datacommons.db differ