-
Notifications
You must be signed in to change notification settings - Fork 36
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add tests for calculated obsCollection #1373
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -186,6 +186,11 @@ func FetchContainedIn( | |
var err error | ||
if store.BtGroup != nil { | ||
readCollectionCache := false | ||
childPlaces, err = FetchChildPlaces( | ||
ctx, store, metadata, httpClient, remoteMixer, ancestor, childType) | ||
if err != nil { | ||
return nil, err | ||
} | ||
if queryDate != "" { | ||
result = &pbv2.ObservationResponse{ | ||
ByVariable: map[string]*pbv2.VariableObservation{}, | ||
|
@@ -210,6 +215,10 @@ func FetchContainedIn( | |
obsByEntity := result.ByVariable[variable].ByEntity | ||
data, ok := btData[variable] | ||
if !ok || data == nil { | ||
// If data is missing, attach empty child places. | ||
for _, place := range childPlaces { | ||
obsByEntity[place] = &pbv2.EntityObservation{} | ||
} | ||
Comment on lines
+218
to
+221
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. feels like this should be part of Flume logic. Right now Flume does not return place with no data. This is a behavior discrepancy. I actually think it makes sense to not have an entry for a child place if there is no data in all scenarios. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should I add this to the flume instead? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I still don't think we should add places with no data. Imagine fetching cities in USA for a custom DC variable which has data for 10 places. That would have thousands of empty entries. I think we should gvc and understand why the child places are needed.. |
||
continue | ||
} | ||
cohorts := data.SourceCohorts | ||
|
@@ -257,11 +266,6 @@ func FetchContainedIn( | |
} | ||
} | ||
if !readCollectionCache { | ||
childPlaces, err = FetchChildPlaces( | ||
ctx, store, metadata, httpClient, remoteMixer, ancestor, childType) | ||
if err != nil { | ||
return nil, err | ||
} | ||
totalSeries := len(variables) * len(childPlaces) | ||
if totalSeries > MaxSeries { | ||
return nil, status.Errorf( | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,225 @@ | ||
{ | ||
"by_variable": { | ||
"Count_Person_Female": { | ||
"by_entity": { | ||
"wikidataId/Q159477": { | ||
"ordered_facets": [ | ||
{ | ||
"observations": [ | ||
{ | ||
"date": "1975", | ||
"value": 6170 | ||
}, | ||
{ | ||
"date": "1980", | ||
"value": 13856 | ||
}, | ||
{ | ||
"date": "1985", | ||
"value": 21711 | ||
}, | ||
{ | ||
"date": "1995", | ||
"value": 48659 | ||
}, | ||
{ | ||
"date": "2005", | ||
"value": 75313 | ||
} | ||
], | ||
"obs_count": 5, | ||
"earliest_date": "1975", | ||
"latest_date": "2005" | ||
} | ||
] | ||
}, | ||
"wikidataId/Q170024": { | ||
"ordered_facets": [ | ||
{ | ||
"observations": [ | ||
{ | ||
"date": "1975", | ||
"value": 17307 | ||
}, | ||
{ | ||
"date": "1980", | ||
"value": 27393 | ||
}, | ||
{ | ||
"date": "1985", | ||
"value": 40165 | ||
}, | ||
{ | ||
"date": "1995", | ||
"value": 59342 | ||
}, | ||
{ | ||
"date": "2005", | ||
"value": 80621 | ||
} | ||
], | ||
"obs_count": 5, | ||
"earliest_date": "1975", | ||
"latest_date": "2005" | ||
} | ||
] | ||
}, | ||
"wikidataId/Q175021": { | ||
"ordered_facets": [ | ||
{ | ||
"observations": [ | ||
{ | ||
"date": "1975", | ||
"value": 2645 | ||
}, | ||
{ | ||
"date": "1980", | ||
"value": 4466 | ||
}, | ||
{ | ||
"date": "1985", | ||
"value": 7626 | ||
}, | ||
{ | ||
"date": "1995", | ||
"value": 14414 | ||
}, | ||
{ | ||
"date": "2005", | ||
"value": 19075 | ||
} | ||
], | ||
"obs_count": 5, | ||
"earliest_date": "1975", | ||
"latest_date": "2005" | ||
} | ||
] | ||
}, | ||
"wikidataId/Q187712": { | ||
"ordered_facets": [ | ||
{ | ||
"observations": [ | ||
{ | ||
"date": "1975", | ||
"value": 56754 | ||
}, | ||
{ | ||
"date": "1980", | ||
"value": 120160 | ||
}, | ||
{ | ||
"date": "1985", | ||
"value": 185783 | ||
}, | ||
{ | ||
"date": "1995", | ||
"value": 291719 | ||
}, | ||
{ | ||
"date": "2005", | ||
"value": 472665 | ||
} | ||
], | ||
"obs_count": 5, | ||
"earliest_date": "1975", | ||
"latest_date": "2005" | ||
} | ||
] | ||
}, | ||
"wikidataId/Q188810": { | ||
"ordered_facets": [ | ||
{ | ||
"observations": [ | ||
{ | ||
"date": "1975", | ||
"value": 27535 | ||
}, | ||
{ | ||
"date": "1980", | ||
"value": 56532 | ||
}, | ||
{ | ||
"date": "1985", | ||
"value": 86466 | ||
}, | ||
{ | ||
"date": "1995", | ||
"value": 148478 | ||
}, | ||
{ | ||
"date": "2005", | ||
"value": 273339 | ||
} | ||
], | ||
"obs_count": 5, | ||
"earliest_date": "1975", | ||
"latest_date": "2005" | ||
} | ||
] | ||
}, | ||
"wikidataId/Q4091": { | ||
"ordered_facets": [ | ||
{ | ||
"observations": [ | ||
{ | ||
"date": "1975", | ||
"value": 6683 | ||
}, | ||
{ | ||
"date": "1980", | ||
"value": 11393 | ||
}, | ||
{ | ||
"date": "1985", | ||
"value": 18343 | ||
}, | ||
{ | ||
"date": "1995", | ||
"value": 30414 | ||
}, | ||
{ | ||
"date": "2005", | ||
"value": 47114 | ||
} | ||
], | ||
"obs_count": 5, | ||
"earliest_date": "1975", | ||
"latest_date": "2005" | ||
} | ||
] | ||
}, | ||
"wikidataId/Q613": { | ||
"ordered_facets": [ | ||
{ | ||
"observations": [ | ||
{ | ||
"date": "1975", | ||
"value": 54366 | ||
}, | ||
{ | ||
"date": "1980", | ||
"value": 88587 | ||
}, | ||
{ | ||
"date": "1985", | ||
"value": 123609 | ||
}, | ||
{ | ||
"date": "1995", | ||
"value": 211211 | ||
}, | ||
{ | ||
"date": "2005", | ||
"value": 332148 | ||
} | ||
], | ||
"obs_count": 5, | ||
"earliest_date": "1975", | ||
"latest_date": "2005" | ||
} | ||
] | ||
} | ||
} | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We don't want to always fetch this since the obsCollection collection could already exists from BT cache.