diff --git a/cumulus_library_data_metrics/__init__.py b/cumulus_library_data_metrics/__init__.py index 15ce0cd..8cc7399 100644 --- a/cumulus_library_data_metrics/__init__.py +++ b/cumulus_library_data_metrics/__init__.py @@ -1,3 +1,3 @@ """Data Metrics study for Cumulus Library""" -__version__ = "5.0.0" +__version__ = "5.0.1" diff --git a/cumulus_library_data_metrics/c_system_use/c_system_use.jinja b/cumulus_library_data_metrics/c_system_use/c_system_use.jinja index 0aee100..2b85dd0 100644 --- a/cumulus_library_data_metrics/c_system_use/c_system_use.jinja +++ b/cumulus_library_data_metrics/c_system_use/c_system_use.jinja @@ -5,8 +5,8 @@ CREATE TABLE data_metrics__count_c_system_use_{{ src|lower }}_{{ field|lower }} WITH src_status AS {{ utils.extract_status(src) }}, -{% if category_system %} -src_categories AS {{ utils.extract_codes(src, 'category', system=category_system, is_array=true) }}, +{% if use_category %} +src_categories AS {{ utils.extract_codes(src, cat_field, system=cat_systems, is_array=true) }}, {% endif %} src_systems_flat AS ( @@ -57,7 +57,7 @@ simplified AS ( {{ utils.get_date_string(dates, 'year') }} AS "year", {% endif %} - {% if category_system %} + {% if use_category %} {{ utils.array_to_string('src_categories.codes') }} AS category, {% endif %} @@ -69,7 +69,7 @@ simplified AS ( LEFT JOIN src_status ON src.id = src_status.id - {% if category_system %} + {% if use_category %} LEFT JOIN src_categories ON src.id = src_categories.id {% endif %} @@ -80,7 +80,7 @@ simplified AS ( {% call utils.make_counts('simplified', output_mode) %} - {% if category_system %} + {% if use_category %} category, {% endif %} diff --git a/cumulus_library_data_metrics/c_system_use/c_system_use.py b/cumulus_library_data_metrics/c_system_use/c_system_use.py index e1c9474..73ad867 100644 --- a/cumulus_library_data_metrics/c_system_use/c_system_use.py +++ b/cumulus_library_data_metrics/c_system_use/c_system_use.py @@ -18,22 +18,10 @@ def make_table(self, **kwargs) -> None: self.queries.append(self.render_sql(self.name, system_names=systems.NAMES, **kwargs)) def add_metric_queries(self) -> None: - self.make_table( - src="Observation", - field="code", - category_system=systems.OBSERVATION_CATEGORY, - ) - self.make_table( - src="Observation", - field="valueCodeableConcept", - category_system=systems.OBSERVATION_CATEGORY, - ) + self.make_table(src="Observation", field="code", use_category=True) + self.make_table(src="Observation", field="valueCodeableConcept", use_category=True) self.make_table(src="AllergyIntolerance", field="code") - self.make_table( - src="Condition", - field="code", - category_system=systems.CONDITION_CATEGORY, - ) + self.make_table(src="Condition", field="code", use_category=True) self.make_table(src="Device", field="type") self.make_table(src="DiagnosticReport", field="code") self.make_table(src="DocumentReference", field="type") diff --git a/cumulus_library_data_metrics/resource_info.py b/cumulus_library_data_metrics/resource_info.py index 6337dde..f6afd82 100644 --- a/cumulus_library_data_metrics/resource_info.py +++ b/cumulus_library_data_metrics/resource_info.py @@ -9,26 +9,40 @@ }, "Condition": { "cat_field": "category", - "cat_systems": [systems.CONDITION_CATEGORY], + "cat_systems": [ + # https://hl7.org/fhir/us/core/stu4/ValueSet-us-core-condition-category.html + systems.CONDITION_CATEGORY, + [systems.USCORE_CONDITION_CATEGORY, ["health-concern"]], + [systems.SNOMED, ["16100001"]], + ], }, "DiagnosticReport": { "cat_field": "category", - "cat_systems": [systems.LOINC, systems.DIAGNOSTIC_SECTION], + # http://hl7.org/fhir/us/core/STU4/StructureDefinition-us-core-diagnosticreport-lab.html + # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-diagnosticreport-category.html + "cat_systems": [ + [systems.DIAGNOSTIC_SECTION, ["LAB"]], # for labs + [systems.LOINC, ["LP29684-5", "LP29708-2", "LP7839-6"]], # for notes + ], }, "DocumentReference": { "cat_field": "category", + # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-documentreference-category.html "cat_systems": [systems.USCORE_DOCREF_CATEGORY], }, "Encounter": { "cat_field": "type", + # https://hl7.org/fhir/us/core/STU4/ValueSet-us-core-encounter-type.html "cat_systems": [systems.CPT, systems.SNOMED], }, "MedicationRequest": { "cat_field": "category", + # http://hl7.org/fhir/R4/valueset-medicationrequest-category.html "cat_systems": [systems.MEDREQ_CATEGORY], }, "Observation": { "cat_field": "category", + # https://www.hl7.org/fhir/R4/valueset-observation-category.html "cat_systems": [systems.OBSERVATION_CATEGORY], }, } diff --git a/cumulus_library_data_metrics/systems.py b/cumulus_library_data_metrics/systems.py index cebe7a0..f53779f 100644 --- a/cumulus_library_data_metrics/systems.py +++ b/cumulus_library_data_metrics/systems.py @@ -16,6 +16,7 @@ DIAGNOSTIC_SECTION = "http://terminology.hl7.org/CodeSystem/v2-0074" MEDREQ_CATEGORY = "http://terminology.hl7.org/CodeSystem/medicationrequest-category" OBSERVATION_CATEGORY = "http://terminology.hl7.org/CodeSystem/observation-category" +USCORE_CONDITION_CATEGORY = "http://hl7.org/fhir/us/core/CodeSystem/condition-category" USCORE_DOCREF_CATEGORY = "http://hl7.org/fhir/us/core/CodeSystem/us-core-documentreference-category" # Object Identifiers (OIDs). diff --git a/cumulus_library_data_metrics/utils.jinja b/cumulus_library_data_metrics/utils.jinja index 7e887fe..adb1951 100644 --- a/cumulus_library_data_metrics/utils.jinja +++ b/cumulus_library_data_metrics/utils.jinja @@ -1,13 +1,30 @@ --- Extracts the codes from a codeableConcept and returns a select with (id, codes). -{% macro extract_codes(src, src_field, system=null, is_array=false) -%} -( - -- Support both a list of systems or a single system +{% macro _compare_systems(system) -%} + {#- Support both a list of systems or a single system #} {%- if system is string %} - {%- set system_list = "('" + system + "')" %} + c.coding.system = '{{ system }}' {%- elif system %} - {%- set system_list = "('" + system|join("', '") + "')" %} + ( + FALSE -- just here so rest of these can start with OR + {%- for sys_info in system %} + {#- sys_info can either by a string or a list of string then codes #} + {%- if sys_info is string %} + OR c.coding.system = '{{ sys_info }}' + {%- else %} + OR ( + c.coding.system = '{{ sys_info[0] }}' + AND c.coding.code in ('{{ sys_info[1]|join("', '") }}') + ) + {%- endif %} + {%- endfor %} + ) + {%- else %} + TRUE {%- endif %} +{%- endmacro %} +-- Extracts the codes from a codeableConcept and returns a select with (id, codes). +{% macro extract_codes(src, src_field, system=null, is_array=false) -%} +( -- Flatten codings of the provided system and recombine the codes as a list SELECT id, @@ -21,9 +38,8 @@ {%- endif %} WHERE c.coding.code IS NOT NULL - {% if system_list %} - AND c.coding.system in {{ system_list }} - {% endif %} + AND {{ _compare_systems(system) }} + GROUP BY id ) {%- endmacro %} @@ -32,13 +48,6 @@ -- Extracts the codes from a codeableConcept and returns a flat select with (id, code). {% macro extract_codes_flat(src, src_field, system=null, is_array=false) -%} ( - -- Support both a list of systems or a single system - {%- if system is string %} - {%- set system_list = "('" + system + "')" %} - {%- elif system %} - {%- set system_list = "('" + system|join("', '") + "')" %} - {%- endif %} - -- Flatten codings of the provided system and recombine the codes as a list SELECT id, @@ -53,9 +62,7 @@ {%- endif %} WHERE c.coding.code IS NOT NULL - {% if system_list %} - AND c.coding.system in {{ system_list }} - {% endif %} + AND {{ _compare_systems(system) }} ) {%- endmacro %} diff --git a/tests/data/c_resource_count/general/condition/0.ndjson b/tests/data/c_resource_count/general/condition/0.ndjson index 3326597..d1d6fb0 100644 --- a/tests/data/c_resource_count/general/condition/0.ndjson +++ b/tests/data/c_resource_count/general/condition/0.ndjson @@ -2,8 +2,8 @@ {"resourceType": "Condition", "id": "onsetDateTime", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "onsetDateTime": "2023-10-04", "onsetPeriod": {"start": "2002-02-02"}} {"resourceType": "Condition", "id": "onsetPeriod", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "onsetPeriod": {"start": "2023-10-04"}} {"resourceType": "Condition", "id": "no-date", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}]} -{"resourceType": "Condition", "id": "multiple-categories", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}, {"coding": [{"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"} -{"resourceType": "Condition", "id": "multiple-codings", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2022-11-28"} +{"resourceType": "Condition", "id": "multiple-categories", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}, {"coding": [{"code": "health-concern", "system": "http://hl7.org/fhir/us/core/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"} +{"resourceType": "Condition", "id": "multiple-codings", "category": [{"coding": [{"code": "16100001", "system": "http://snomed.info/sct"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2022-11-28"} {"resourceType": "Condition", "id": "unrelated-category", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "nope"}]}, {"coding": [{"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-10-28"} {"resourceType": "Condition", "id": "unrelated-coding", "category": [{"coding": [{"code": "encounter-diagnosis", "system": "nope"}, {"code": "problem-list-item", "system": "http://terminology.hl7.org/CodeSystem/condition-category"}]}], "recordedDate": "2023-11-28"} {"resourceType": "Condition", "id": "no-category", "recordedDate": "2023-10-03"} diff --git a/tests/data/c_resource_count/general/diagnosticreport/0.ndjson b/tests/data/c_resource_count/general/diagnosticreport/0.ndjson index 381aede..51bc6e6 100644 --- a/tests/data/c_resource_count/general/diagnosticreport/0.ndjson +++ b/tests/data/c_resource_count/general/diagnosticreport/0.ndjson @@ -1,6 +1,6 @@ -{"resourceType": "DiagnosticReport", "id": "with-date", "category": [{"coding": [{"code": "CT", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}], "effectiveDateTime": "2023-10-04"} +{"resourceType": "DiagnosticReport", "id": "with-date", "category": [{"coding": [{"code": "LAB", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}], "effectiveDateTime": "2023-10-04"} {"resourceType": "DiagnosticReport", "id": "with-issued", "issued": "2021-02-15", "status": "partial"} -{"resourceType": "DiagnosticReport", "id": "no-date", "category": [{"coding": [{"code": "CT", "system": "http://loinc.org"}]}]} -{"resourceType": "DiagnosticReport", "id": "unrelated-category", "category": [{"coding": [{"code": "1234", "system": "nope"}]}]} +{"resourceType": "DiagnosticReport", "id": "no-date", "category": [{"coding": [{"code": "LP29684-5", "system": "http://loinc.org"}]}]} +{"resourceType": "DiagnosticReport", "id": "unrelated-category", "category": [{"coding": [{"code": "12345-6", "system": "http://loinc.org"}, {"code": "CT", "system": "http://terminology.hl7.org/CodeSystem/v2-0074"}]}]} {"resourceType": "DiagnosticReport", "id": "no-category", "effectivePeriod": {"start": "2022-09-04"}} {"resourceType": "DiagnosticReport", "id": "nothing"} diff --git a/tests/data/c_resource_count/general/expected_condition_month.csv b/tests/data/c_resource_count/general/expected_condition_month.csv index 5b0fbcd..cc21849 100644 --- a/tests/data/c_resource_count/general/expected_condition_month.csv +++ b/tests/data/c_resource_count/general/expected_condition_month.csv @@ -2,9 +2,9 @@ cnt,category,month,status 2,encounter-diagnosis,2023-10,cumulus__none 1,problem-list-item,2023-11,cumulus__none 1,problem-list-item,2023-10,cumulus__none -1,encounter-diagnosis; problem-list-item,2023-11,cumulus__none -1,encounter-diagnosis; problem-list-item,2022-11,cumulus__none +1,encounter-diagnosis; health-concern,2023-11,cumulus__none 1,encounter-diagnosis,cumulus__none,cumulus__none 1,encounter-diagnosis,2023-11,cumulus__none 1,cumulus__none,cumulus__none,cumulus__none 1,cumulus__none,2023-10,cumulus__none +1,16100001; problem-list-item,2022-11,cumulus__none diff --git a/tests/data/c_resource_count/general/expected_condition_year.csv b/tests/data/c_resource_count/general/expected_condition_year.csv index d155ff7..b3321d8 100644 --- a/tests/data/c_resource_count/general/expected_condition_year.csv +++ b/tests/data/c_resource_count/general/expected_condition_year.csv @@ -1,8 +1,8 @@ cnt,category,year,status 3,encounter-diagnosis,2023,cumulus__none 2,problem-list-item,2023,cumulus__none -1,encounter-diagnosis; problem-list-item,2023,cumulus__none -1,encounter-diagnosis; problem-list-item,2022,cumulus__none +1,encounter-diagnosis; health-concern,2023,cumulus__none 1,encounter-diagnosis,cumulus__none,cumulus__none 1,cumulus__none,cumulus__none,cumulus__none 1,cumulus__none,2023,cumulus__none +1,16100001; problem-list-item,2022,cumulus__none diff --git a/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv b/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv index 8404cb1..ee24ccd 100644 --- a/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv +++ b/tests/data/c_resource_count/general/expected_diagnosticreport_month.csv @@ -2,5 +2,5 @@ cnt,category,month,status 2,cumulus__none,cumulus__none,cumulus__none 1,cumulus__none,2022-09,cumulus__none 1,cumulus__none,2021-02,partial -1,CT,cumulus__none,cumulus__none -1,CT,2023-10,cumulus__none +1,LP29684-5,cumulus__none,cumulus__none +1,LAB,2023-10,cumulus__none diff --git a/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv b/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv index 1031de2..75af675 100644 --- a/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv +++ b/tests/data/c_resource_count/general/expected_diagnosticreport_year.csv @@ -2,5 +2,5 @@ cnt,category,year,status 2,cumulus__none,cumulus__none,cumulus__none 1,cumulus__none,2022,cumulus__none 1,cumulus__none,2021,partial -1,CT,cumulus__none,cumulus__none -1,CT,2023,cumulus__none +1,LP29684-5,cumulus__none,cumulus__none +1,LAB,2023,cumulus__none