diff --git a/pipeline/workflow/aggregation-helper/Dockerfile b/pipeline/workflow/aggregation-helper/Dockerfile index b782018c8..8934dd82a 100644 --- a/pipeline/workflow/aggregation-helper/Dockerfile +++ b/pipeline/workflow/aggregation-helper/Dockerfile @@ -14,13 +14,29 @@ FROM python:3.12-slim +# Copy uv binary +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + # Allow statements and log messages to immediately appear in the logs ENV PYTHONUNBUFFERED True WORKDIR /app -# Copy local code to the container image -COPY main.py . +# Copy dependency definition files to leverage Docker layer caching +COPY ingestion-helper/pyproject.toml ingestion-helper/uv.lock ingestion-helper/__init__.py ./ + +# Install production dependencies (without the project itself) +RUN uv sync --no-dev --no-install-project + +# Copy ingestion-helper code (includes aggregation package) and main.py +COPY ingestion-helper/ . +COPY aggregation-helper/main.py . + +# Install the project itself +RUN uv sync --no-dev + +# Place the virtual environment's bin directory on the PATH +ENV PATH="/app/.venv/bin:$PATH" -# Run the script +# Run the Cloud Run Job script ENTRYPOINT ["python", "main.py"] diff --git a/pipeline/workflow/aggregation-helper/cloudbuild.yaml b/pipeline/workflow/aggregation-helper/cloudbuild.yaml index b3fc0d318..4c3f78cf1 100644 --- a/pipeline/workflow/aggregation-helper/cloudbuild.yaml +++ b/pipeline/workflow/aggregation-helper/cloudbuild.yaml @@ -17,6 +17,7 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: [ 'build', + '-f', 'aggregation-helper/Dockerfile', '-t', '${_AR_REPO_URL}/${_IMAGE_NAME}:${_TAG}', '-t', '${_AR_REPO_URL}/${_IMAGE_NAME}:${_VERSION}', '.' diff --git a/pipeline/workflow/aggregation-helper/main.py b/pipeline/workflow/aggregation-helper/main.py index 7dd68802c..186f66b9b 100644 --- a/pipeline/workflow/aggregation-helper/main.py +++ b/pipeline/workflow/aggregation-helper/main.py @@ -12,20 +12,39 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Aggregation Helper Cloud Run Job skeleton.""" +"""Aggregation Helper Cloud Run Job execution entry point.""" import argparse import json import logging +import os import sys +from aggregation import AggregationOrchestrator + + def main(): logging.basicConfig(level=logging.INFO) - logging.info("Starting Aggregation Helper Job") + logging.info("Starting Aggregation Helper Cloud Run Job...") parser = argparse.ArgumentParser(description="Run aggregation helper job.") - parser.add_argument("--import_list", help="JSON string representing the list of imports to process.") - + parser.add_argument( + "--import_list", + help="JSON string representing the list of imports to process." + ) + parser.add_argument( + "--dry_run", + action="store_true", + default=True, + help="Run in dry-run mode without executing jobs (default: True)." + ) + parser.add_argument( + "--execute", + action="store_false", + dest="dry_run", + help="Disable dry-run mode and execute BigQuery aggregation jobs." + ) + args = parser.parse_args() if not args.import_list: @@ -34,7 +53,7 @@ def main(): try: import_list = json.loads(args.import_list) - logging.info(f"Received import list: {import_list}") + logging.info(f"Received active imports to process: {import_list}") except json.JSONDecodeError as e: logging.error(f"Failed to parse import_list JSON: {e}") sys.exit(1) @@ -43,12 +62,30 @@ def main(): logging.error("Parsed import_list is not a list") sys.exit(1) - # Dummy logic - logging.info("Processing aggregation (dummy)...") - for imp in import_list: - logging.info(f"Processing import: {imp}") - - logging.info("Aggregation Helper Job completed successfully.") + connection_id = os.environ.get("BQ_SPANNER_CONN_ID") + project_id = os.environ.get("PROJECT_ID") + instance_id = os.environ.get("SPANNER_INSTANCE_ID") + database_id = os.environ.get("SPANNER_DATABASE_ID") or os.environ.get("SPANNER_GRAPH_DATABASE_ID") + location = os.environ.get("LOCATION") + + if not all([connection_id, project_id, instance_id, database_id]): + logging.error( + f"Missing required environment variables. connection_id={connection_id}, " + f"project_id={project_id}, instance_id={instance_id}, database_id={database_id}" + ) + sys.exit(1) + + orchestrator = AggregationOrchestrator( + connection_id=connection_id, + project_id=project_id, + instance_id=instance_id, + database_id=database_id, + location=location, + ) + + logging.info(f"Executing AggregationOrchestrator pipeline (dry_run={args.dry_run}) for imports: {import_list}") + orchestrator.run(active_imports=import_list, dry_run=args.dry_run) + logging.info("Aggregation Helper Cloud Run Job completed successfully.") if __name__ == "__main__": main() diff --git a/pipeline/workflow/build-services.yaml b/pipeline/workflow/build-services.yaml index dde23e970..f6bf5f530 100644 --- a/pipeline/workflow/build-services.yaml +++ b/pipeline/workflow/build-services.yaml @@ -36,7 +36,7 @@ steps: - id: 'build-aggregation-helper' name: 'gcr.io/cloud-builders/gcloud' - args: ['builds', 'submit', 'aggregation-helper', '--config', 'aggregation-helper/cloudbuild.yaml', '--substitutions', '_AR_REPO_URL=${_AR_REPO_URL},_VERSION=${_VERSION}'] + args: ['builds', 'submit', '.', '--config', 'aggregation-helper/cloudbuild.yaml', '--substitutions', '_AR_REPO_URL=${_AR_REPO_URL},_VERSION=${_VERSION}'] dir: 'pipeline/workflow' waitFor: ['-'] diff --git a/pipeline/workflow/ingestion-helper/aggregation/README.md b/pipeline/workflow/ingestion-helper/aggregation/README.md new file mode 100644 index 000000000..53e96feda --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/README.md @@ -0,0 +1,101 @@ +# Aggregations + +This module orchestrates the execution of Data Commons aggregations through BigQuery Federation. The aggregations include place rollups, statistical variable aggregations, entity aggregations, linked edges, and metadata summaries. + +## Core Concepts + +* **Sequential Stages**: Aggregations are executed sequentially by their `stage` number (e.g., Stage 1 steps are guaranteed to complete before Stage 2 steps begin). This is useful when later steps depend on the output of earlier ones. +* **Parallel Execution**: All aggregation steps configured in the same stage are executed in parallel to maximize performance. +* **Per-Import Isolation**: Aggregations run independently for each active import dataset. + +--- + +## Configuration Guide (`configs/*.yaml`) + +The aggregation pipeline is configured via YAML files in the `configs/` directory (`place.yaml`, `statvar.yaml`, `common.yaml`, etc.). Each file defines a top-level `calculations:` list. + +### Common Configuration Fields +Every calculation step supports these common fields: +* `type` (string, Required): The type of calculation step (e.g. `PLACE_AGGREGATION`, `STAT_VAR_AGGREGATION`, `LINKED_EDGES`). +* `stage` (integer, Optional, default: 1): The sequential stage number. Steps in lower stages are guaranteed to finish before higher stages start. +* `input_imports` (list of strings, Required): The list of import names this step applies to. Use `["*"]` (wildcard) to apply the step to **all** active imports. +* `output_import` (string, Optional): The output import dataset name to write aggregated observations under. +* `disabled` (boolean, Optional, default: false): Set to `true` to temporarily disable a step without deleting it. + +--- + +### Supported Calculation Types + +#### 1. Place Aggregation (`PLACE_AGGREGATION`) +Aggregates and rolls up statistical data from a smaller place type (source) to a larger place type (destination). +* **Fields**: + * `from_place_types` (string, Required): The source place type (e.g., `State`). + * `to_place_types` (string, Required): The destination place type (e.g., `Country`). + * `allow_multiple_to_places` (boolean, Optional, default: false): Allows mapping to multiple parent places if true. +* **Example**: + ```yaml + - type: PLACE_AGGREGATION + stage: 1 + input_imports: + - CensusACS5YearSurvey + output_import: CensusACS5YearSurvey_AggCountry + place_aggregation: + from_place_types: State + to_place_types: Country + ``` + +#### 2. Statistical Variable Aggregation (`STAT_VAR_AGGREGATION`) +Aggregates raw statistical variables into a summarized ancestor variable. +* **Fields**: + * `ancestor_sv_id` (string, Required): The ID of the parent/summary statistical variable (e.g., `Count_Person`). + * `source_sv_ids` (list of strings, Required): The list of individual statistical variables to sum up. + * `skip_all_sources_present_check` (boolean, Optional, default: false): If true, aggregates even if some source variables are missing. +* **Example**: + ```yaml + - type: STAT_VAR_AGGREGATION + stage: 2 + input_imports: + - CensusACS5YearSurvey + output_import: CensusACS5YearSurvey_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person + source_sv_ids: + - Count_Person_Male + - Count_Person_Female + ``` + +#### 3. Common Aggregations (`LINKED_EDGES`, `PROVENANCE_SUMMARY`, `STAT_VAR_GROUPS`) +Common graph structure, lineage, and UI group hierarchy rollups defined in `common.yaml`. +* **Example**: + ```yaml + - type: LINKED_EDGES + stage: 1 + input_imports: + - "*" + ``` + +--- + +## Local Configuration Validation + +The orchestrator strictly validates configuration files against `schema.json`. If there is any syntax error, type mismatch, or missing required field, validation will fail. + +### Running the Validator Locally +You can validate all configuration files locally using the built-in CLI tool: + +1. **Navigate to the ingestion-helper root**: + ```bash + cd pipeline/workflow/ingestion-helper + ``` +2. **Run the validator**: + ```bash + python3 -m aggregation.validator + + # Sample output: + # Validating 7 configuration file(s) in 'aggregation/configs' against 'schema.json'... + # ✓ common.yaml (3 calculation steps) + # ✓ place.yaml (16 calculation steps) + # ✓ statvar.yaml (21 calculation steps) + # [SUCCESS] All 7 configuration file(s) passed validation! (64 calculation steps total) + ``` diff --git a/pipeline/workflow/ingestion-helper/aggregation/__init__.py b/pipeline/workflow/ingestion-helper/aggregation/__init__.py index b85759cf5..74be5ed23 100644 --- a/pipeline/workflow/ingestion-helper/aggregation/__init__.py +++ b/pipeline/workflow/ingestion-helper/aggregation/__init__.py @@ -24,6 +24,8 @@ from .place_aggregation_generator import PlaceAggregationGenerator from .stat_var_group_generator import StatVarGroupGenerator from .stat_var_calculation_generator import StatVarCalculationGenerator +from .orchestrator import AggregationOrchestrator +from .validator import validate_config __all__ = [ 'BigQueryExecutor', @@ -32,5 +34,7 @@ 'StatVarAggregator', 'PlaceAggregationGenerator', 'StatVarGroupGenerator', - 'StatVarCalculationGenerator' + 'StatVarCalculationGenerator', + 'AggregationOrchestrator', + 'validate_config', ] diff --git a/pipeline/workflow/ingestion-helper/aggregation/bq_executor.py b/pipeline/workflow/ingestion-helper/aggregation/bq_executor.py index 58f4f992e..010e8606b 100644 --- a/pipeline/workflow/ingestion-helper/aggregation/bq_executor.py +++ b/pipeline/workflow/ingestion-helper/aggregation/bq_executor.py @@ -37,8 +37,14 @@ def __init__(self, self.location = location # TODO: Remove run_sequential logic once DCP migrates to async execution. self.run_sequential = run_sequential - self.client = bigquery.Client(project=self.project_id, - location=self.location) + self._client: Optional[bigquery.Client] = None + + @property + def client(self) -> bigquery.Client: + """Lazily initializes and returns the BigQuery client.""" + if self._client is None: + self._client = bigquery.Client(project=self.project_id, location=self.location) + return self._client def get_spanner_destination_uri(self) -> str: """Returns the Spanner destination URI for EXPORT DATA.""" diff --git a/pipeline/workflow/ingestion-helper/aggregation/configs/common.yaml b/pipeline/workflow/ingestion-helper/aggregation/configs/common.yaml new file mode 100644 index 000000000..27cc90dac --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/configs/common.yaml @@ -0,0 +1,21 @@ +calculations: + # Generates linkedContainedInPlace, linkedMemberOf, etc. + - name: "Global: ContainedInPlace & MemberOf Graph Linked Edges" + type: LINKED_EDGES + stage: 1 + input_imports: + - "*" + + # Generates summary statistics in the Cache table + - name: "Global: Cache Provenance & Lineage Summary" + type: PROVENANCE_SUMMARY + stage: 1 + input_imports: + - "*" + + # Generates the Statistical Variable hierarchy/verticals + - name: "Global: StatVar Group Hierarchy & Verticals" + type: STAT_VAR_GROUPS + stage: 1 + input_imports: + - "*" diff --git a/pipeline/workflow/ingestion-helper/aggregation/configs/entity.yaml b/pipeline/workflow/ingestion-helper/aggregation/configs/entity.yaml new file mode 100644 index 000000000..0f970c8c6 --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/configs/entity.yaml @@ -0,0 +1,190 @@ +calculations: + + # Earthquakes + - name: "EarthquakeUSGS: Earthquake Event Aggregation" + type: ENTITY_AGGREGATION + input_imports: + - EarthquakeUSGS + output_import: EarthquakeUSGS_Agg + entity_aggregation: + entity_types: + - EarthquakeEvent + location_props: + - affectedPlace + date_prop: occurrenceTime + agg_date_formats: + - YYYY + - YYYY-MM + constraints: + - "magnitude: [3 - M]" + - "magnitude: [4 - M]" + - "magnitude: [5 - M]" + - "magnitude: [6 - M]" + - "magnitude: [7 - M]" + - "magnitude: [8 - M]" + - "magnitude: [9 - M]" + - "magnitude: [3 4 M]" + - "magnitude: [4 5 M]" + - "magnitude: [5 6 M]" + - "magnitude: [6 7 M]" + - "magnitude: [7 8 M]" + - "magnitude: [8 9 M]" + + # Fires + - name: "FireFAMWEB: Wildland Fire Event Aggregation" + type: ENTITY_AGGREGATION + input_imports: + - FireFAMWEB + output_import: FireFAMWEB_Agg + entity_aggregation: + entity_types: + - WildlandFireEvent + location_props: + - location + date_prop: discoveryDate + agg_date_formats: + - YYYY + + - name: "FireWFIGS: Fire Incident Event Aggregation" + type: ENTITY_AGGREGATION + input_imports: + - FireWFIGS + output_import: FireWFIGS_Agg + entity_aggregation: + entity_types: + - FireIncidentComplexEvent + - PrescribedFireEvent + - WildlandFireEvent + location_props: + - location + date_prop: discoveryDate + agg_date_formats: + - YYYY-MM + - YYYY + + - name: "NASA_VIIRSActiveFiresEvents: Fire Event S2 Cell Aggregation" + type: ENTITY_AGGREGATION + # Aggregate event counts for S2-cells of level 13 + input_imports: + - NASA_VIIRSActiveFiresEvents + output_import: NASA_VIIRSActiveFiresEvents_Agg + entity_aggregation: + entity_types: + - FireEvent + location_props: + - affectedPlace + date_prop: startDate + agg_date_formats: + - YYYY + - YYYY-MM + + # Storms + - name: "StormNOAA: Storm & Extreme Weather Event Aggregation" + type: ENTITY_AGGREGATION + input_imports: + - StormNOAA + output_import: StormNOAA_Agg + entity_aggregation: + location_props: + - affectedPlace + date_prop: startDate + agg_date_formats: + - YYYY + - YYYY-MM + entity_types: + - MarineHighWindEvent + - HighWindEvent + - HeavyRainEvent + - DebrisFlowEvent + - DenseSmokeEvent + - TropicalDepressionEvent + - LandslideEvent + - DustDevilEvent + - WinterStormEvent + - MarineHailEvent + - TsunamiEvent + - BlizzardEvent + - RipCurrentEvent + - HeavySnowEvent + - ExtremeColdWindChillEvent + - MarineDenseFogEvent + - TropicalStormEvent + - StormSurgeTideEvent + - NorthernLightsEvent + - FrostFreezeEvent + - WildfireEvent + - SleetEvent + - ColdWindChillEvent + - MarineStrongWindEvent + - LightningEvent + - SneakerwaveEvent + - AstronomicalLowTideEvent + - MarineTropicalStormEvent + - VolcanicAshEvent + - CoastalFloodEvent + - GeoCoordinates + - HeavyWindEvent + - IceStormEvent + - StrongWindEvent + - WinterWeatherEvent + - VolcanicAshfallEvent + - ThunderstormWindEvent + - FlashFloodEvent + - HeatEvent + - DustStormEvent + - ExcessiveHeatEvent + - HailEvent + - AvalancheEvent + - MarineThunderstormWindEvent + - LakeshoreFloodEvent + - DroughtEvent + - HighSurfEvent + - HurricaneTyphoonEvent + - LakeEffectSnowEvent + - WaterspoutEvent + - MarineTropicalDepressionEvent + - SeicheEvent + - HighSnowEvent + - DenseFogEvent + - HurricaneEvent + - MarineHurricaneTyphoonEvent + - FloodEvent + - FunnelCloudEvent + - FreezingFogEvent + - TornadoEvent + - MarineLightningEvent + - StormEpisode + + # Floods + - name: "DynamicWorld_FloodEvents: Flood Event S2 Cell Aggregation" + type: ENTITY_AGGREGATION + # Aggregate event counts for S2-cells of level 13 + input_imports: + - DynamicWorld_FloodEvents + output_import: DynamicWorld_FloodEvents_Agg + entity_aggregation: + entity_types: + - FloodEvent + location_props: + - affectedPlace + date_prop: startDate + agg_date_formats: + - YYYY + - YYYY-MM + + # Heat/Cold Temperature Events counts + - name: "TemperatureEvents: Heat & Cold Event Aggregation" + type: ENTITY_AGGREGATION + input_imports: + - TemperatureEvents + output_import: TemperatureEvents_Agg + entity_aggregation: + entity_types: + - HeatTemperatureEvent + - ColdTemperatureEvent + location_props: + - affectedPlace + date_prop: startDate + agg_date_formats: + - YYYY + - YYYY-MM diff --git a/pipeline/workflow/ingestion-helper/aggregation/configs/place.yaml b/pipeline/workflow/ingestion-helper/aggregation/configs/place.yaml new file mode 100644 index 000000000..1072f43cd --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/configs/place.yaml @@ -0,0 +1,163 @@ +calculations: + - name: "CensusACS5YearSurvey: State -> Country Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - CensusACS5YearSurvey + output_import: CensusACS5YearSurvey_AggCountry + place_aggregation: + from_place_types: State + to_place_types: Country + + - name: "CensusSAHIE: State -> Country Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - CensusSAHIE + output_import: CensusSAHIE_AggCountry + place_aggregation: + from_place_types: State + to_place_types: Country + + - name: "CDCMortality: County -> State Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - CDCMortality + output_import: CDCMortality_AggState + place_aggregation: + from_place_types: County + to_place_types: State + + - name: "CDCMortality: State -> Country Place Rollup" + type: PLACE_AGGREGATION + stage: 2 + input_imports: + - CDCMortality_AggState + output_import: CDCMortality_AggState_AggCountry + place_aggregation: + from_place_types: State + to_place_types: Country + + - name: "DEA_ARCOS: County -> State Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - DEA_ARCOS + output_import: DEA_ARCOS_AggState + place_aggregation: + from_place_types: County + to_place_types: State + + - name: "DEA_ARCOS: State -> Country Place Rollup" + type: PLACE_AGGREGATION + stage: 2 + input_imports: + - DEA_ARCOS_AggState + output_import: DEA_ARCOS_AggState_AggCountry + place_aggregation: + from_place_types: State + to_place_types: Country + + - name: "EPA_EJSCREEN: CensusBlockGroup -> CensusTract Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - EPA_EJSCREEN + output_import: EPA_EJSCREEN_AggCensusTract + place_aggregation: + from_place_types: CensusBlockGroup + to_place_types: CensusTract + + - name: "EPA_EJSCREEN: CensusTract -> County Place Rollup" + type: PLACE_AGGREGATION + stage: 2 + input_imports: + - EPA_EJSCREEN_AggCensusTract + output_import: EPA_EJSCREEN_AggCensusTract_AggCounty + place_aggregation: + from_place_types: CensusTract + to_place_types: County + + - name: "DeepSolar: CensusBlockGroup -> CensusTract Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - DeepSolar + output_import: DeepSolar_AggCensusTract + place_aggregation: + from_place_types: CensusBlockGroup + to_place_types: CensusTract + + - name: "DeepSolar: CensusTract -> County Place Rollup" + type: PLACE_AGGREGATION + stage: 2 + input_imports: + - DeepSolar_AggCensusTract + output_import: DeepSolar_AggCensusTract_AggCounty + place_aggregation: + from_place_types: CensusTract + to_place_types: County + + - name: "EPA_GHGRP: EpaReportingFacility -> County Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - EPA_GHGRP + output_import: EPA_GHGRP_AggCounty + place_aggregation: + from_place_types: EpaReportingFacility + to_place_types: County + + - name: "EPA_GHGRP: EpaReportingFacility -> CensusZipCodeTabulationArea Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - EPA_GHGRP + output_import: EPA_GHGRP_AggCensusZipCodeTabulationArea + place_aggregation: + from_place_types: EpaReportingFacility + to_place_types: CensusZipCodeTabulationArea + + - name: "EPA_GHGRP: County -> State Place Rollup" + type: PLACE_AGGREGATION + stage: 2 + input_imports: + - EPA_GHGRP_AggCounty + output_import: EPA_GHGRP_AggCounty_AggState + place_aggregation: + from_place_types: County + to_place_types: State + + - name: "RFF_USGridGeo: GeoGrid -> County Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - RFF_USGridGeo_WeatherVariabilityForecast + output_import: RFF_USGridGeo_WeatherVariabilityForecast_AggCounty + place_aggregation: + from_place_types: GeoGridPlace_0.25Deg + to_place_types: County + allow_multiple_to_places: true + + - name: "India_RBIStateDomesticProduct: State -> Country Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - India_RBIStateDomesticProduct + output_import: India_RBIStateDomesticProduct_AggCountry + place_aggregation: + from_place_types: State + to_place_types: Country + allow_multiple_to_places: true + + - name: "India_RBIStateDomesticProduct: StatVarAgg State -> Country Place Rollup" + type: PLACE_AGGREGATION + stage: 1 + input_imports: + - India_RBIStateDomesticProduct_StatVarAgg + output_import: India_RBIStateDomesticProduct_StatVarAgg_AggCountry + place_aggregation: + from_place_types: State + to_place_types: Country + allow_multiple_to_places: true diff --git a/pipeline/workflow/ingestion-helper/aggregation/configs/statvar.yaml b/pipeline/workflow/ingestion-helper/aggregation/configs/statvar.yaml new file mode 100644 index 000000000..f3992282a --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/configs/statvar.yaml @@ -0,0 +1,1914 @@ +calculations: + + # Health Insurance Coverage + - name: "CensusACS5YearSurvey: Health Insurance SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_HealthInsurance_StatVarAgg + stat_var_aggregation: + aggregations: + # 1. Census Table B18135 includes insurance coverage by disability-status and + # age (https://data.census.gov/cedsci/table?tid=ACSDT5Y2019.B18135). + # We use that to compute total and by disability-status. + # + # Total uninsured + - ancestor_sv_id: Count_Person_NoHealthInsurance + source_sv_ids: + # No Disability + - dc/y0dvhk0sggzef + - dc/kdg05h55y45y6 + - dc/9drszqwd2nef7 + # With Disability + - dc/bew8kj6l7tv93 + - dc/96dqj47csvmy8 + - dc/qr4s77egv27q2 + # Total insured + - ancestor_sv_id: Count_Person_WithHealthInsurance + source_sv_ids: + # No Disability + - dc/32mhsxvq7qsm4 + - dc/2s2dkbb7gz038 + - dc/dc8vqzkx18x0c + # With Disability + - dc/s5efzs4x817p5 + - dc/9j39148yn79zf + - dc/p3v76jcvdx919 + # Uninsured with no disability + # The following 4 aggregations come from parts of the above two aggregations. + - ancestor_sv_id: Count_Person_NoHealthInsurance_NoDisability + source_sv_ids: + # No Disability + - dc/y0dvhk0sggzef + - dc/kdg05h55y45y6 + - dc/9drszqwd2nef7 + # Uninsured with disability + - ancestor_sv_id: Count_Person_NoHealthInsurance_WithDisability + source_sv_ids: + # With Disability + - dc/bew8kj6l7tv93 + - dc/96dqj47csvmy8 + - dc/qr4s77egv27q2 + # Insured with no disability + - ancestor_sv_id: Count_Person_WithHealthInsurance_NoDisability + source_sv_ids: + # No Disability + - dc/32mhsxvq7qsm4 + - dc/2s2dkbb7gz038 + - dc/dc8vqzkx18x0c + # Insured with disability + - ancestor_sv_id: Count_Person_WithHealthInsurance_WithDisability + source_sv_ids: + # With Disability + - dc/s5efzs4x817p5 + - dc/9j39148yn79zf + - dc/p3v76jcvdx919 + # + # 2. Census Table B27001 includes insurance coverage by age and gender + # breakdown (https://data.census.gov/cedsci/table?tid=ACSDT5Y2019.B27001). We + # use that to compute gender breakdown. + # + - ancestor_sv_id: Count_Person_Female_NoHealthInsurance + source_sv_ids: + - dc/jx2q10tbnwhf3 + - dc/g1957rtkkrnpg + - dc/kbd1j4tv1r383 + - dc/xkfbhgbnzhm8d + - dc/001qmh9sdqzeb + - dc/qhwj3qx90h4pd + - dc/jydkzj9v0yb2 + - dc/qb6th3c5tcd92 + - dc/zj152fzswgrw6 + - ancestor_sv_id: Count_Person_Male_NoHealthInsurance + source_sv_ids: + - dc/fwqlkjjch0dg5 + - dc/1qdvp8x0t3l11 + - dc/33z5t6xqjepp1 + - dc/260gxgmeglpb + - dc/l8j7ggjt8yx8b + - dc/4rdkjc2485w42 + - dc/nn4xz2lv98h5c + - dc/ywjqcxc244sx2 + - dc/j88b1fy7045j1 + - ancestor_sv_id: Count_Person_Female_WithHealthInsurance + source_sv_ids: + - dc/jjhxlkp3qk2m6 + - dc/lwqlh3vtpzw51 + - dc/1h5c2xbpd0edb + - dc/q4lk959j0bgq + - dc/1hs676pfqqthd + - dc/kkbq8kzlm6tw4 + - dc/y9xxbzfr8yb06 + - dc/whh1y53fj4p69 + - dc/965mdb0c8tef9 + - ancestor_sv_id: Count_Person_Male_WithHealthInsurance + source_sv_ids: + - dc/ve6s1v1eyrbt7 + - dc/1n4mf17q78xtd + - dc/zyhtyhs9efej3 + - dc/y6tzpetgq89v4 + - dc/js56cehzj7wf4 + - dc/yrmj9vp8b8n1d + - dc/wp3txgscfen9c + - dc/b3n86k0h3h2w + - dc/j8se7wv2gbdfd + # dc/g/Person_HealthInsurance-NoPrivateHealthInsurance + # Level 3 + - ancestor_sv_id: Count_Person_NoPrivateHealthInsurance + source_sv_ids: + # Population: 26 - 34 Years, Female, No Private Health Insurance + - dc/0kyv91d5902mg + # Population: 75 Years or More, Male, No Private Health Insurance + - dc/1ld59l0sp7z85 + # Population: 6 Years or Less, Male, No Private Health Insurance + - dc/27q8fsx1rm8e1 + # Population: 55 - 64 Years, Female, No Private Health Insurance + - dc/2v3y7wgqrzm5f + # Population: 65 - 74 Years, Female, No Private Health Insurance + - dc/388xq8h8hdhyh + # Population: 18 - 24 Years, Female, No Private Health Insurance + - dc/40rjc7178vkrd + # Population: 25 - 34 Years, Male, No Private Health Insurance + - dc/4qk2e1c4zhsr8 + # Population: 45 - 54 Years, Female, No Private Health Insurance + - dc/60mq1lysnc39g + # Population: 35 - 44 Years, Female, No Private Health Insurance + - dc/9cwrwgd2t1r95 + # Population: 19 - 25 Years, Male, No Private Health Insurance + - dc/9xxzewchgl6bd + # Population: 6 - 17 Years, Female, No Private Health Insurance + - dc/b9s112lccg17g + # Population: 26 - 34 Years, Male, No Private Health Insurance + - dc/dsmsdwbj1yqf9 + # Population: 45 - 54 Years, Male, No Private Health Insurance + - dc/hycewjthgy8w + # Population: 6 Years or Less, Female, No Private Health Insurance + - dc/n0e5f7zjsbedg + # Population: 6 - 18 Years, Male, No Private Health Insurance + - dc/njeqykf73dng2 + # Population: 19 - 25 Years, Female, No Private Health Insurance + - dc/nl2wdjffxpdx3 + # Population: 25 - 34 Years, Female, No Private Health Insurance + - dc/s11tttbvg3dzh + # Population: 55 - 64 Years, Male, No Private Health Insurance + - dc/v6d0158t101s7 + # Population: 65 - 74 Years, Male, No Private Health Insurance + - dc/v80klfl3e9m7f + # Population: 6 - 18 Years, Female, No Private Health Insurance + - dc/w77cv2c32fck7 + # Population: 18 - 24 Years, Male, No Private Health Insurance + - dc/wch97nj65s85f + # Population: 75 Years or More, Female, No Private Health Insurance + - dc/wrz9m375r1yy + # Population: 35 - 44 Years, Male, No Private Health Insurance + - dc/wyfvejhn9fe7 + # Population: 6 - 17 Years, Male, No Private Health Insurance + - dc/yfmzp444fj5r5 + # dc/g/Person_HealthInsurance-NoPublicHealthInsurance + # Level 3 + - ancestor_sv_id: Count_Person_NoPublicHealthInsurance + source_sv_ids: + # Population: 26 - 34 Years, Female, No Public Health Insurance + - dc/0l10mjl6jcb43 + # Population: 65 - 74 Years, Male, No Public Health Insurance + - dc/34kcegr057z16 + # Population: 19 - 25 Years, Female, No Public Health Insurance + - dc/3l656p2tqt8m9 + # Population: 45 - 54 Years, Female, No Public Health Insurance + - dc/81xpj06sdg2kd + # Population: 26 - 34 Years, Male, No Public Health Insurance + - dc/cpcn8j7mw3tx3 + # Population: 35 - 44 Years, Male, No Public Health Insurance + - dc/e922mrcqvemm6 + # Population: 19 - 25 Years, Male, No Public Health Insurance + - dc/edh0y8vjc9y55 + # Population: 6 - 18 Years, Male, No Public Health Insurance + - dc/efg2r5lbj2rs + # Population: 65 - 74 Years, Female, No Public Health Insurance + - dc/fz5dn8yl6m516 + # Population: 45 - 54 Years, Male, No Public Health Insurance + - dc/g9qp5cejpyhq5 + # Population: 75 Years or More, Male, No Public Health Insurance + - dc/gc7lcsrrv1rm5 + # Population: 6 - 18 Years, Female, No Public Health Insurance + - dc/grj8xq0mmr602 + # Population: 35 - 44 Years, Female, No Public Health Insurance + - dc/h8123v7kk1xr + # Population: 55 - 64 Years, Male, No Public Health Insurance + - dc/jrh189de6x30h + # Population: 6 - 17 Years, Male, No Public Health Insurance + - dc/ljcepzzm38m7h + # Population: 18 - 24 Years, Female, No Public Health Insurance + - dc/mhfv62j3s22rc + # Population: 18 - 24 Years, Male, No Public Health Insurance + - dc/p8pjrf6ffdeb + # Population: 75 Years or More, Female, No Public Health Insurance + - dc/qtmer196cydk + # Population: 6 Years or Less, Female, No Public Health Insurance + - dc/r4yy8tmxk2pb + # Population: 25 - 34 Years, Male, No Public Health Insurance + - dc/t8p05s93nvjq1 + # Population: 6 - 17 Years, Female, No Public Health Insurance + - dc/v1ez3g1cnv2b2 + # Population: 6 Years or Less, Male, No Public Health Insurance + - dc/v7xhmvehm80ph + # Population: 55 - 64 Years, Female, No Public Health Insurance + - dc/vxecjxxbmhy43 + # Population: 25 - 34 Years, Female, No Public Health Insurance + - dc/yjxmx3n02dx08 + # dc/g/Person_HealthInsurance-WithOneTypeOfHealthInsurance + # Level 1 + - ancestor_sv_id: Count_Person_WithOneTypeOfHealthInsurance + source_sv_ids: + # Population: 18 Years or Less, With One Type of Health Insurance + - dc/3jbqx1kf03nd4 + # Population: 18 - 34 Years, With One Type of Health Insurance + - dc/3s8t9m1k1pgw5 + # Population: 35 - 64 Years, With One Type of Health Insurance + - dc/hcvy4028mk048 + # Population: 19 Years or Less, With One Type of Health Insurance + - dc/sjvp50nwdlpmf + # Population: 19 - 34 Years, With One Type of Health Insurance + - dc/thcbepzex9zd9 + # Population: 65 Years or More, With One Type of Health Insurance + - dc/v1w52jrtvw6m2 + # dc/g/Person_HealthInsurance-WithPublicCoverage + # Level 3 + - ancestor_sv_id: Count_Person_WithPublicCoverage + source_sv_ids: + # Population: 18 Years or Less, With Disability, With Public Coverage + - dc/0f66x8keewypb + # Population: 19 Years or Less, With Disability, With Public Coverage + - dc/1s4eg9f8m6328 + # Population: 19 Years or Less, No Disability, With Public Coverage + - dc/35b9fkeh2qlf + # Population: 65 Years or More, No Disability, With Public Coverage + - dc/7cj9xhqe5262d + # Population: 19 - 64 Years, No Disability, With Public Coverage + - dc/84cnddqz2egl6 + # Population: 65 Years or More, With Disability, With Public Coverage + - dc/nqtzsgv2g5h09 + # Population: 18 - 64 Years, With Disability, With Public Coverage + - dc/qekes8xvrvz18 + # Population: 18 Years or Less, No Disability, With Public Coverage + - dc/r7975mdk9v7kh + # Population: 18 - 64 Years, No Disability, With Public Coverage + - dc/t41xetm1g0q6h + # Population: 19 - 64 Years, With Disability, With Public Coverage + - dc/zkcy0f52ewh04 + + # + # Ability to speak English + # ------------------------ + # + # We aggregate abilityToSpeakEnglish over nativity (2) and languageSpokenAtHome + # (4) for a total of 8 source SVs from Census Table B16005 + # (https://data.census.gov/cedsci/table?tid=ACSDT5Y2019.B16005) + # + # NOTE: Values SpeakEnglishLessThanVeryWell and SpeakEnglishVeryWell already + # have population values from a different table. + # + - name: "CensusACS5YearSurvey: Ability To Speak English SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_AbilityToSpeakEnglish_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_SpeakEnglishNotAtAll + source_sv_ids: + - dc/edfjy64gmxf6f + - dc/0165994v0l8fh + - dc/0ltdeskbvkn3d + - dc/ves02m300f6lc + - dc/vh52n3qgw0kt + - dc/6yzk8t4e9t6v6 + - dc/0kx32ff9c6d79 + - dc/7cxlkzf56zk26 + - ancestor_sv_id: Count_Person_SpeakEnglishNotWell + source_sv_ids: + - dc/f8qh4hp1830dg + - dc/3f6nmhf0zhtd4 + - dc/l1zr0zx1yje61 + - dc/zh4sly3kmwbwg + - dc/e4jl4qe9xc4n9 + - dc/6rzmvxpgqlww6 + - dc/wf1rm5zx34dtg + - dc/4xykjw3v6n4t3 + - ancestor_sv_id: Count_Person_SpeakEnglishWell + source_sv_ids: + - dc/4n1wg2c7hjem + - dc/hbqhxpwsdvyj8 + - dc/ewchtlvjlljk5 + - dc/7st63yed0l4x + - dc/vx85c3xzwg981 + - dc/92mnzpclp7tbd + - dc/yx0mdj3dnk3mb + - dc/1x48nzrne2f88 + + # + # In Armed Forces + # --------------- + # + # Census Table B23001 includes count of people in armed forces (in which case + # In labor Forces is implied) broken down by gender and age. From that we + # aggregate to total and by gender. + # (https://data.census.gov/cedsci/table?tid=ACSDT5Y2019.B23001) + # + - name: "CensusACS5YearSurvey: In Armed Forces SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_InArmedForces_StatVarAgg + stat_var_aggregation: + aggregations: + # Women in armed forces + - ancestor_sv_id: Count_Person_Female_InArmedForces + source_sv_ids: + - dc/173smewzddlb + - dc/fjd99f6xm3mz3 + - dc/n9wqebpd8jyhc + - dc/7tej0l9s6x2rh + - dc/hqz1tck7sdrm2 + - dc/tmw7srr91bj41 + - dc/lzzl0rx7m1yf9 + - dc/8d182v9j04mg4 + - dc/lb3lb4mg82mph + - dc/g2m31qc7q1x64 + # Men in armed forces + - ancestor_sv_id: Count_Person_Male_InArmedForces + source_sv_ids: + - dc/vp1gqv00d2ql3 + - dc/04nv03cmv6btf + - dc/rhgnypl16nh6c + - dc/mtqbxrg29f64h + - dc/gd0jznk5n2kb3 + - dc/4f6fv8bdstp3b + - dc/rlrwr4f7r4pg7 + - dc/x63tjfw28tzvc + - dc/fpx513jvf4xed + - dc/2zj8jthd4f563 + # Armed forces population, from combining the above two sets of SVs. + - ancestor_sv_id: Count_Person_InArmedForces + source_sv_ids: + # Female + - dc/173smewzddlb + - dc/fjd99f6xm3mz3 + - dc/n9wqebpd8jyhc + - dc/7tej0l9s6x2rh + - dc/hqz1tck7sdrm2 + - dc/tmw7srr91bj41 + - dc/lzzl0rx7m1yf9 + - dc/8d182v9j04mg4 + - dc/lb3lb4mg82mph + - dc/g2m31qc7q1x64 + # Male + - dc/vp1gqv00d2ql3 + - dc/04nv03cmv6btf + - dc/rhgnypl16nh6c + - dc/mtqbxrg29f64h + - dc/gd0jznk5n2kb3 + - dc/4f6fv8bdstp3b + - dc/rlrwr4f7r4pg7 + - dc/x63tjfw28tzvc + - dc/fpx513jvf4xed + - dc/2zj8jthd4f563 + + # Education - CensusACS5YearSurvey + - name: "CensusACS5YearSurvey: Education SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_Education_StatVarAgg + stat_var_aggregation: + aggregations: + # dc/g/Person_EducationalAttainment-5ThAnd6ThGrade + # Level 1 + - ancestor_sv_id: Count_Person_Years25Onwards_EducationalAttainment_5ThAnd6ThGrade + source_sv_ids: + # Population: 5th And 6th Grade, Female + - Count_Person_25OrMoreYears_EducationalAttainment5ThAnd6ThGrade_Female + # Population: 5th And 6th Grade, Male + - Count_Person_25OrMoreYears_EducationalAttainment5ThAnd6ThGrade_Male + # dc/g/Person_EducationalAttainment-7ThAnd8ThGrade + # Level 1 + - ancestor_sv_id: Count_Person_Years25Onwards_EducationalAttainment_7ThAnd8ThGrade + source_sv_ids: + # Population: 7th And 8th Grade, Female + - Count_Person_25OrMoreYears_EducationalAttainment7ThAnd8ThGrade_Female + # Population: 7th And 8th Grade, Male + - Count_Person_25OrMoreYears_EducationalAttainment7ThAnd8ThGrade_Male + # dc/g/Person_EducationalAttainment-9ThTo12ThGradeNoDiploma + # Level 3 + - ancestor_sv_id: Count_Person_EducationalAttainment_9ThTo12ThGradeNoDiploma + source_sv_ids: + # Population: 18 - 24 Years, 9th To 12th Grade No Diploma, Female + - dc/g0y4zefyr10n7 + # Population: 18 - 24 Years, 9th To 12th Grade No Diploma, Male + - dc/0kptwgt2r2j97 + # Population: 25 - 34 Years, 9th To 12th Grade No Diploma, Female + - dc/4f3r13e4gkhq6 + # Population: 25 - 34 Years, 9th To 12th Grade No Diploma, Male + - dc/tcv30t9ydkldc + # Population: 35 - 44 Years, 9th To 12th Grade No Diploma, Female + - dc/7k5lylyz4l2th + # Population: 35 - 44 Years, 9th To 12th Grade No Diploma, Male + - dc/nrmj89cggsfg6 + # Population: 45 - 64 Years, 9th To 12th Grade No Diploma, Female + - dc/m3g8esl121v73 + # Population: 45 - 64 Years, 9th To 12th Grade No Diploma, Male + - dc/3hfl00nblgvg5 + # Population: 65 Years or More, 9th To 12th Grade No Diploma, Female + - dc/5sps7rmylm73b + # Population: 65 Years or More, 9th To 12th Grade No Diploma, Male + - dc/07hctc6f9e2k9 + # dc/g/Person_EducationalAttainment-LessThan9ThGrade + # Level 3 + - ancestor_sv_id: Count_Person_EducationalAttainment_LessThan9ThGrade + source_sv_ids: + # Population: 18 - 24 Years, Less Than 9th Grade, Female + - dc/935ftchsz31b1 + # Population: 18 - 24 Years, Less Than 9th Grade, Male + - dc/n6v6b8vh8jnd5 + # Population: 25 - 34 Years, Less Than 9th Grade, Female + - dc/dr8wcfnpxyrj7 + # Population: 25 - 34 Years, Less Than 9th Grade, Male + - dc/k8g20v452g617 + # Population: 35 - 44 Years, Less Than 9th Grade, Female + - dc/y051838618t9c + # Population: 35 - 44 Years, Less Than 9th Grade, Male + - dc/cpx5ll8vth2n7 + # Population: 45 - 64 Years, Less Than 9th Grade, Female + - dc/b841qxd3smpt + # Population: 45 - 64 Years, Less Than 9th Grade, Male + - dc/5vhc0r8m4x5g9 + # Population: 65 Years or More, Less Than 9th Grade, Female + - dc/292723k92k5tb + # Population: 65 Years or More, Less Than 9th Grade, Male + - dc/68pblb53csteb + # dc/g/Person_EducationalAttainment-LessThanHighSchoolDiploma + # Level 3 + - ancestor_sv_id: Count_Person_EducationalAttainment_LessThanHighSchoolDiploma + source_sv_ids: + # Population: Less Than High School Diploma, Male, Two or More Races + - dc/3g5g2k9n19l45 + # Population: Less Than High School Diploma, Male, Asian Alone + - dc/z3z44q7edbyp7 + # Population: Less Than High School Diploma, Female, Hispanic or Latino + - dc/0p89e7q72edr1 + # Population: Less Than High School Diploma, Male, White Alone + - dc/70kcr1l5jldsb + # Population: Less Than High School Diploma, Female, Asian Alone + - dc/nbfp13v6t9v87 + # Population: Less Than High School Diploma, Male, Hispanic or Latino + - dc/szpdtvh1v6p49 + # Population: Less Than High School Diploma, Male, Some Other Race Alone + - dc/pyflt85w79j35 + # Population: Less Than High School Diploma, Female, White Alone Not Hispanic or Latino + - dc/vhxsz5bplmef4 + # Population: Less Than High School Diploma, Male, Black or African American Alone + - dc/k51n39gvd22t5 + # Population: Less Than High School Diploma, Female, Black or African American Alone + - dc/m1trmbvh8sygd + # Population: Less Than High School Diploma, Male, Native Hawaiian or Other Pacific Islander Alone + - dc/7n0468s56spt8 + # Population: Less Than High School Diploma, Male, American Indian or Alaska Native Alone + - dc/5e4qydg4slyvd + # Population: Less Than High School Diploma, Female, Native Hawaiian or Other Pacific Islander Alone + - dc/y95cbfgtlqrmb + # Population: Less Than High School Diploma, Female, Some Other Race Alone + - dc/p7978m8tbsns6 + # Population: Less Than High School Diploma, Female, American Indian or Alaska Native Alone + - dc/lyzcfc37eet64 + # Population: Less Than High School Diploma, Female, White Alone + - dc/m4q40nln3qms + # Population: Less Than High School Diploma, Male, White Alone Not Hispanic or Latino + - dc/yfrrvevrmyr74 + # Population: Less Than High School Diploma, Female, Two or More Races + - dc/29l3m1z7d3n7c + # dc/g/Person_EducationalAttainment-NurseryTo4ThGrade + # Level 1 + - ancestor_sv_id: Count_Person_Years25Onwards_EducationalAttainment_NurseryTo4ThGrade + source_sv_ids: + # Population: Nursery To 4th Grade, Female + - Count_Person_25OrMoreYears_EducationalAttainmentNurseryTo4ThGrade_Female + # Population: Nursery To 4th Grade, Male + - Count_Person_25OrMoreYears_EducationalAttainmentNurseryTo4ThGrade_Male + # dc/g/Person_EducationalAttainment-SomeCollegeNoDegree + # Level 3 + - ancestor_sv_id: Count_Person_EducationalAttainment_SomeCollegeNoDegree + source_sv_ids: + # Population: 18 - 24 Years, Some College No Degree, Female + - dc/x8sydp80d61v9 + # Population: 18 - 24 Years, Some College No Degree, Male + - dc/bxs5x91p5jbf + # Population: 25 - 34 Years, Some College No Degree, Female + - dc/z8j7q9303cpx8 + # Population: 25 - 34 Years, Some College No Degree, Male + - dc/d4p0m2p51v7d3 + # Population: 35 - 44 Years, Some College No Degree, Female + - dc/66nscflhly519 + # Population: 35 - 44 Years, Some College No Degree, Male + - dc/q4eclxrhd5mrd + # Population: 45 - 64 Years, Some College No Degree, Female + - dc/dsmks2epl3ve + # Population: 45 - 64 Years, Some College No Degree, Male + - dc/6ms49l0q9tq44 + # Population: 65 Years or More, Some College No Degree, Female + - dc/c0vyt03x0h0v4 + # Population: 65 Years or More, Some College No Degree, Male + - dc/5s08n6kdt26db + - ancestor_sv_id: Count_Person_EducationalAttainment_1StTo12ThGrade + source_sv_ids: + - Count_Person_EducationalAttainment1StGrade + - Count_Person_EducationalAttainment2NdGrade + - Count_Person_EducationalAttainment3RdGrade + - Count_Person_EducationalAttainment4ThGrade + - Count_Person_EducationalAttainment5ThGrade + - Count_Person_EducationalAttainment6ThGrade + - Count_Person_EducationalAttainment7ThGrade + - Count_Person_EducationalAttainment8ThGrade + - Count_Person_EducationalAttainment9ThGrade + - Count_Person_EducationalAttainment10ThGrade + - Count_Person_EducationalAttainment11ThGrade + - Count_Person_EducationalAttainment12ThGradeNoDiploma + + # The following 9 aggregations are: SomeCollegeOrAssociatesDegree, by race. + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_AmericanIndianOrAlaskaNativeAlone + source_sv_ids: + # Population: Some College or Associates Degree, Female, American Indian or Alaska Native Alone + - dc/mqxdr821c7kw3 + # Population: Some College or Associates Degree, Male, American Indian or Alaska Native Alone + - dc/lyt8y97bbrkpc + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_AsianAlone + source_sv_ids: + # Population: Some College or Associates Degree, Female, Asian Alone + - dc/whn99h1l0xgth + # Population: Some College or Associates Degree, Male, Asian Alone + - dc/fkvnj4rlrs84f + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_BlackOrAfricanAmericanAlone + source_sv_ids: + # Population: Some College or Associates Degree, Female, Black or African American Alone + - dc/56md3ndhrmvm7 + # Population: Some College or Associates Degree, Male, Black or African American Alone + - dc/e7h67vn5w4g13 + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_HispanicOrLatino + source_sv_ids: + # Population: Some College or Associates Degree, Female, Hispanic or Latino + - dc/3w039ndqy7qv1 + # Population: Some College or Associates Degree, Male, Hispanic or Latino + - dc/7xf6mm0sg9y18 + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_NativeHawaiianOrOtherPacificIslanderAlone + source_sv_ids: + # Population: Some College or Associates Degree, Female, Native Hawaiian or Other Pacific Islander Alone + - dc/kpcfmb6lp3zpd + # Population: Some College or Associates Degree, Male, Native Hawaiian or Other Pacific Islander Alone + - dc/r3l3rfl1ms85f + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_SomeOtherRaceAlone + source_sv_ids: + # Population: Some College or Associates Degree, Female, Some Other Race Alone + - dc/epw58ne8mytn5 + # Population: Some College or Associates Degree, Male, Some Other Race Alone + - dc/t0mcpxqgr2lm9 + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_TwoOrMoreRaces + source_sv_ids: + # Population: Some College or Associates Degree, Female, Two or More Races + - dc/q98jxycvs422f + # Population: Some College or Associates Degree, Male, Two or More Races + - dc/zjlfv8d8v14f8 + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_WhiteAlone + source_sv_ids: + # Population: Some College or Associates Degree, Female, White Alone + - dc/9sneyc8lpk8dc + # Population: Some College or Associates Degree, Male, White Alone + - dc/d2ct8qmvcct81 + - ancestor_sv_id: Count_Person_25OrMoreYears_SomeCollegeOrAssociatesDegree_WhiteAloneNotHispanicOrLatino + source_sv_ids: + # Population: Some College or Associates Degree, Female, White Alone Not Hispanic or Latino + - dc/bqy52h7y4nq34 + # Population: Some College or Associates Degree, Male, White Alone Not Hispanic or Latino + - dc/dc9v9h3q8l8n7 + + # Education - ACSED5YrSurvey + - name: "CensusSAHIE_AggCountry: Health Insurance SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - ACSED5YrSurvey + output_import: CensusSAHIE_AggCountry_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Parent_Occupation_Management_Business_Science_Arts + source_sv_ids: + # Count of Parent: 16 Years or More, Civilian, Public School, Employed, in Labor Force, Management, Business, Science, And Arts Occupations + - dc/zvmdlctml84sb + - ancestor_sv_id: Count_Parent_Occupation_Natural_Resources_Construction_Maintenance + source_sv_ids: + # Count of Parent: 16 Years or More, Civilian, Public School, Employed, in Labor Force, Natural Resources, Construction, And Maintenance Occupations + - dc/fsrwm5cy84lmf + - ancestor_sv_id: Count_Parent_Occupation_Production_Transportation_Material_Moving + source_sv_ids: + # Count of Parent: 16 Years or More, Civilian, Public School, Employed, in Labor Force, Production, Transportation, And Material Moving Occupations + - dc/jbyb43crb1t37 + - ancestor_sv_id: Count_Parent_Occupation_Sales_Office + source_sv_ids: + # Count of Parent: 16 Years or More, Civilian, Public School, Employed, in Labor Force, Sales And Office Occupations + - dc/p029jbzf00pw6 + - ancestor_sv_id: Count_Parent_Occupation_Service + source_sv_ids: + # Count of Parent: 16 Years or More, Civilian, Public School, Employed, in Labor Force, Service Occupations + - dc/bstxmnb4k1wrb + + # Demographics + - name: "CensusACS5YearSurvey: Demographics SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_Demographics_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_85OrMoreYears + source_sv_ids: + - Count_Person_85OrMoreYears_Male + - Count_Person_85OrMoreYears_Female + - ancestor_sv_id: Count_Person_0To4Years + source_sv_ids: + - Count_Person_0To4Years_Male + - Count_Person_0To4Years_Female + + # Agriculture + - name: "USDA_AgricultureCensus: Agriculture Producer SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - USDA_AgricultureCensus + output_import: USDA_AgricultureCensus_Agriculture_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_Producer + source_sv_ids: + - Count_Person_Producer_BlackOrAfricanAmericanAlone + - Count_Person_Producer_AmericanIndianOrAlaskaNativeAlone + - Count_Person_Producer_NativeHawaiianOrOtherPacificIslanderAlone + - Count_Person_Producer_WhiteAlone + - Count_Person_Producer_TwoOrMoreRaces + - Count_Person_Producer_HispanicOrLatino + - Count_Person_Producer_AsianAlone + skip_all_sources_present_check: true + + # + # Employment + # ------------------------ + + - name: "CensusACS5YearSurvey: Employment & Industry SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_Employment_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSAgricultureForestryFishingHunting + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Agriculture, Forestry, Fishing And Hunting (NAICS/11) + - dc/scgpgdxgx9tr1 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Agriculture, Forestry, Fishing And Hunting (NAICS/11) + - dc/klz4pcgfe44rb + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSMiningQuarryingOilGasExtraction + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Mining, Quarrying, And Oil And Gas Extraction (NAICS/21) + - dc/jcq9116jwjvs3 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Mining, Quarrying, And Oil And Gas Extraction (NAICS/21) + - dc/xkywpwshj3ttb + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSUtilities + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Utilities (NAICS/22) + - dc/ewhc3j80tx5z1 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Utilities (NAICS/22) + - dc/hjepq5mk3yvy9 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSConstruction + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Construction (NAICS/23) + - dc/1h1rnmvtj706b + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Construction (NAICS/23) + - dc/0llhlrmmm5ce2 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSManufacturing + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Manufacturing (NAICS/31-33) + - dc/xsqgzwf1h8f33 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Manufacturing (NAICS/31-33) + - dc/ypmrvdtd4e159 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSWholesaleTrade + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Wholesale Trade (NAICS/42) + - dc/v2q0shf60bejd + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Wholesale Trade (NAICS/42) + - dc/3cn2vsmlfsfv1 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSRetailTrade + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Retail Trade (NAICS/44-45) + - dc/qcfd17k1c3vm9 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Retail Trade (NAICS/44-45) + - dc/3hh7cp878qgjb + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSTransportationWarehousing + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Transportation And Warehousing (NAICS/48-49) + - dc/cq266pvtkbll4 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Transportation And Warehousing (NAICS/48-49) + - dc/j9f2bf9fb5y1g + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSInformation + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Information (NAICS/51) + - dc/c9yj0kkxpf8ff + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Information (NAICS/51) + - dc/20cmnywrsexnc + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSFinanceInsurance + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Finance And Insurance (NAICS/52) + - dc/e7yf34hemnde9 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Finance And Insurance (NAICS/52) + - dc/3gcb6txyx3t88 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_Count_Establishment_NAICSRealEstateRentalLeasing + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Real Estate And Rental And Leasing (NAICS/53) + - dc/l292my8mjkns + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Real Estate And Rental And Leasing (NAICS/53) + - dc/4zgbhyej0lnd + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSProfessionalScientificTechnicalServices + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Professional, Scientific, And Technical Services (NAICS/54) + - dc/lc629dk4yeg4b + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Professional, Scientific, And Technical Services (NAICS/54) + - dc/gtbj7nt7jh608 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSManagementOfCompaniesEnterprises + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Management of Companies And Enterprises (NAICS/55) + - dc/zt2qhhtnjm4qh + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Management of Companies And Enterprises (NAICS/55) + - dc/z9qrge1jf1k3h + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSAdministrativeSupportWasteManagementRemediationServices + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Administrative And Support And Waste Management Services (NAICS/56) + - dc/ff2f4my72zx0c + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Administrative And Support And Waste Management Services (NAICS/56) + - dc/cmy92ne7lqr4d + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSEducationalServices + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Educational Services (NAICS/61) + - dc/1wr6sspk9nrjc + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Educational Services (NAICS/61) + - dc/xjvlbg6jjb6q8 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSHealthCareSocialAssistance + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Health Care And Social Assistance (NAICS/62) + - dc/nmd0vjps6psb2 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Health Care And Social Assistance (NAICS/62) + - dc/4gr6y83p0hvj + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSArtsEntertainmentRecreation + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Arts, Entertainment, And Recreation (NAICS/71) + - dc/9hrltcsyj2mcb + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Arts, Entertainment, And Recreation (NAICS/71) + - dc/0xmrxl7ztb4e7 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSAccommodationFoodServices + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Accommodation And Food Services (NAICS/72) + - dc/rydmsxhz7pxh1 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Accommodation And Food Services (NAICS/72) + - dc/pn22zf92qr31f + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSOtherServices + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Other Services, Except Public Administration (NAICS/81) + - dc/cyc1f26msdct4 + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Other Services, Except Public Administration (NAICS/81) + - dc/ns2e3khxs6gw7 + - ancestor_sv_id: Count_Person_16OrMoreYears_Civilian_Employed_InLaborForce_NAICSPublicAdministration + source_sv_ids: + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Female, Public Administration (NAICS/92) + - dc/ylj424exymm1b + # Population: 16 Years or More, Civilian, Employed, in Labor Force, Male, Public Administration (NAICS/92) + - dc/589p0gc36qem5 + + # Census Table B23001 includes count of employed civilians broken down by gender + # and age. From that we aggregate over age to get count of employed civilians + # for each gender. + # (https://data.census.gov/cedsci/table?tid=ACSDT5Y2019.B23001) + # + # - type: STAT_VAR_AGGREGATION + # type: STAT_VAR_AGGREGATION + # output_import: CensusACS5YearSurvey_Employment_StatVarAgg + # input_imports: + # - CensusACS5YearSurvey + # - CensusACS5YearSurvey_AggCountry + # stat_var_aggregation: + # aggregations: + # # Women Employed + # + # # The ancestor has "armedForcesStatus: dcs:Civilian", but the following sources + # # don't have. Fix it before adding this to aggregation. + # # dc/kz49wc5n3lhpd + # # dc/qcpg8c533syd6 + # # dc/zv26z833d7g3c + # - ancestor_sv_id: Count_Person_Female_Employed + # source_sv_ids: + # - dc/yt1fm72s1y7b7 + # - dc/hj65vwnt5csr7 + # - dc/0sd3x3bb4qet5 + # - dc/j4mzcc63n5zq5 + # - dc/56jhsezrzl049 + # - dc/ggx918j9p5tnf + # - dc/8rw47nf3ngle7 + # - dc/hplj99j7mbfsh + # - dc/ksmx7fwfkm8lb + # - dc/6706lc55kg5d + # - dc/qcpg8c533syd6 + # - dc/zv26z833d7g3c + # - dc/kz49wc5n3lhpd + # } + # + # # The ancestor has "armedForcesStatus: dcs:Civilian", but the following sources + # # don't have. Fix it before adding this to aggregation. + # # dc/5cxs4br0jz02c + # # dc/s909dd4r22fw + # # dc/xepldf55yq6s5 + # + # # Men Employed + # - ancestor_sv_id: Count_Person_Male_Employed + # source_sv_ids: + # - dc/2s6hps4z1qced + # - dc/xvtk9180lme1h + # - dc/twdr4c500yev5 + # - dc/n0btf3nglhqqc + # - dc/7qwvkhr4tsyt8 + # - dc/pkpm179bbh822 + # - dc/gyrw68q6x77l9 + # - dc/ly0fvmlf4mtf5 + # - dc/m020zpvzch2gd + # - dc/sgm602ncby3bf + # - dc/xepldf55yq6s5 + # - dc/s909dd4r22fw + # - dc/5cxs4br0jz02c + # } + + # Crime + - name: "USNationalPrisonerStatistics: Crime & Correctional Facility SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - USNationalPrisonerStatistics + output_import: USNationalPrisonerStatistics_Crime_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_CorrectionalFacilityLocation_OutOfState + source_sv_ids: + - dc/qgv9d3frn35qc + - dc/91vy0sf20wlg9 + + # WithOwnChildrenUnder18. + - name: "CensusACS5YearSurvey_SubjectTables_S1251: Children & Household SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey_SubjectTables_S1251 + output_import: CensusACS5YearSurvey_SubjectTables_S1251_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_WithOwnChildrenUnder18_Female + source_sv_ids: + - Count_Person_WithOwnChildrenUnder18_Female_FamilyHousehold_MarriedInThePast12Months_ResidesInHousehold + - Count_Person_WithOwnChildrenUnder18_Female_FamilyHousehold_DivorcedInThePast12Months_ResidesInHousehold + - ancestor_sv_id: Count_Person_WithOwnChildrenUnder18_Male + source_sv_ids: + - Count_Person_WithOwnChildrenUnder18_Male_FamilyHousehold_MarriedInThePast12Months_ResidesInHousehold + - Count_Person_WithOwnChildrenUnder18_Male_FamilyHousehold_DivorcedInThePast12Months_ResidesInHousehold + - ancestor_sv_id: Count_Person_WithOwnChildrenUnder18 + source_sv_ids: + - Count_Person_WithOwnChildrenUnder18_Female + - Count_Person_WithOwnChildrenUnder18_Male + + # Marriage + - name: "CensusACS5YearSurvey_SubjectTables_S1201: Marital Status SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey_SubjectTables_S1201 + output_import: CensusACS5YearSurvey_SubjectTables_S1201_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_InLaborForce_Divorced + source_sv_ids: + - Count_Person_InLaborForce_Female_Divorced + - Count_Person_InLaborForce_Male_Divorced + - ancestor_sv_id: Count_Person_InLaborForce_NeverMarried + source_sv_ids: + - Count_Person_InLaborForce_Female_NeverMarried + - Count_Person_InLaborForce_Male_NeverMarried + - ancestor_sv_id: Count_Person_InLaborForce_MarriedAndNotSeparated + source_sv_ids: + - Count_Person_InLaborForce_Female_MarriedAndNotSeparated + - Count_Person_InLaborForce_Male_MarriedAndNotSeparated + - ancestor_sv_id: Count_Person_InLaborForce_Widowed + source_sv_ids: + - Count_Person_InLaborForce_Female_Widowed + - Count_Person_InLaborForce_Male_Widowed + + # Employment by business ownership type. + - name: "CensusACS5YearSurvey_SubjectTables_S2408: Business Ownership Employment SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey_SubjectTables_S2408 + output_import: CensusACS5YearSurvey_SubjectTables_S2408_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_PrivatelyOwnedNotForProfitEstablishment_PaidWorker + source_sv_ids: + - Count_Person_PrivatelyOwnedNotForProfitEstablishment_Male_PaidWorker + - Count_Person_PrivatelyOwnedNotForProfitEstablishment_Female_PaidWorker + - ancestor_sv_id: Count_Person_PrivatelyOwnedForProfitEstablishment_PaidWorker + source_sv_ids: + - Count_Person_PrivatelyOwnedForProfitEstablishment_Male_PaidWorker + - Count_Person_PrivatelyOwnedForProfitEstablishment_Female_PaidWorker + + # US Citizen by Naturalization + - name: "CensusACS5YearSurvey_SubjectTables_S0504: Naturalized Citizenship SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey_SubjectTables_S0504 + output_import: CensusACS5YearSurvey_SubjectTables_S0504_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn + source_sv_ids: + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthAfrica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthCaribbean + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthCentralAmericaExceptMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthLatinAmerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthNorthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthNorthernWesternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthOceania + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthSouthCentralAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthSouthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthSouthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthSouthernEasternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn_PlaceOfBirthWesternAsia + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn + source_sv_ids: + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthAfrica + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthCaribbean + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthCentralAmericaExceptMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthLatinAmerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthNorthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthNorthernWesternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthOceania + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthSouthCentralAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthSouthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthSouthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthSouthernEasternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn_PlaceOfBirthWesternAsia + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn + source_sv_ids: + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthAfrica + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthCaribbean + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthCentralAmericaExceptMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthLatinAmerica + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthNorthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthNorthernWesternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthOceania + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthSouthCentralAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthSouthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthSouthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthSouthernEasternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn_PlaceOfBirthWesternAsia + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn + source_sv_ids: + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthAfrica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthCaribbean + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthCentralAmericaExceptMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthLatinAmerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthNorthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthNorthernWesternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthOceania + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthSouthCentralAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthSouthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthSouthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthSouthernEasternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn_PlaceOfBirthWesternAsia + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn + source_sv_ids: + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthAfrica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthCaribbean + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthCentralAmericaExceptMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthLatinAmerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthNorthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthNorthernWesternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthOceania + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthSouthCentralAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthSouthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthSouthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthSouthernEasternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn_PlaceOfBirthWesternAsia + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn + source_sv_ids: + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthAfrica + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthCaribbean + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthCentralAmericaExceptMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthMexico + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthLatinAmerica + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthNorthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthNorthernWesternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthOceania + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthSouthCentralAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthSouthEasternAsia + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthSouthamerica + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthSouthernEasternEurope + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn_PlaceOfBirthWesternAsia + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_USCitizenByNaturalization + source_sv_ids: + - Count_Person_USCitizenByNaturalization_DateOfEntry1990OrEarlier_ForeignBorn + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrEarlier_ForeignBorn + - Count_Person_USCitizenByNaturalization_DateOfEntry2000To2009_ForeignBorn + - Count_Person_USCitizenByNaturalization_DateOfEntry1990To1999_ForeignBorn + - Count_Person_USCitizenByNaturalization_DateOfEntry2000OrLater_ForeignBorn + - Count_Person_USCitizenByNaturalization_DateOfEntry2010OrLater_ForeignBorn + + # HousingUnit HomeValue. + - name: "CensusACS5YearSurvey: Home Value SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_HousingUnit_HomeValue_StatVarAgg + stat_var_aggregation: + aggregations: + # Census ACS 5 year, Count_HousingUnit_HomeValue + - ancestor_sv_id: Count_HousingUnit_HomeValueUpto49999USDollar + source_sv_ids: + - Count_HousingUnit_HomeValueUpto10000USDollar + - Count_HousingUnit_HomeValue10000To14999USDollar + - Count_HousingUnit_HomeValue15000To19999USDollar + - Count_HousingUnit_HomeValue20000To24999USDollar + - Count_HousingUnit_HomeValue25000To29999USDollar + - Count_HousingUnit_HomeValue30000To34999USDollar + - Count_HousingUnit_HomeValue35000To39999USDollar + - Count_HousingUnit_HomeValue40000To49999USDollar + skip_all_sources_present_check: true + - ancestor_sv_id: Count_HousingUnit_HomeValue50000To99999USDollar + source_sv_ids: + - Count_HousingUnit_HomeValue50000To59999USDollar + - Count_HousingUnit_HomeValue60000To69999USDollar + - Count_HousingUnit_HomeValue70000To79999USDollar + - Count_HousingUnit_HomeValue80000To89999USDollar + - Count_HousingUnit_HomeValue90000To99999USDollar + skip_all_sources_present_check: true + - ancestor_sv_id: Count_HousingUnit_HomeValue100000To199999USDollar + source_sv_ids: + - Count_HousingUnit_HomeValue100000To124999USDollar + - Count_HousingUnit_HomeValue125000To149999USDollar + - Count_HousingUnit_HomeValue150000To174999USDollar + - Count_HousingUnit_HomeValue175000To199999USDollar + skip_all_sources_present_check: true + - ancestor_sv_id: Count_HousingUnit_HomeValue200000To299999USDollar + source_sv_ids: + - Count_HousingUnit_HomeValue200000To249999USDollar + - Count_HousingUnit_HomeValue250000To299999USDollar + skip_all_sources_present_check: true + - ancestor_sv_id: Count_HousingUnit_HomeValue300000To499999USDollar + source_sv_ids: + - Count_HousingUnit_HomeValue300000To399999USDollar + - Count_HousingUnit_HomeValue400000To499999USDollar + skip_all_sources_present_check: true + - ancestor_sv_id: Count_HousingUnit_HomeValue500000To999999USDollar + source_sv_ids: + - Count_HousingUnit_HomeValue500000To749999USDollar + - Count_HousingUnit_HomeValue750000To999999USDollar + skip_all_sources_present_check: true + + - name: "CensusACS5YearSurvey: Person Age Groups SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_Person_Age_StatVarAgg + stat_var_aggregation: + aggregations: + # Census ACS 5 year, Age group + - ancestor_sv_id: Count_Person_55To64Years + source_sv_ids: + - Count_Person_55To59Years + - Count_Person_60To61Years + - Count_Person_62To64Years + skip_all_sources_present_check: true + + - name: "CensusACS5YearSurvey: Individual Income SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_Income_StatVarAgg + stat_var_aggregation: + aggregations: + # The following 9 aggregations are: High income (100,000 USD or More), by race. + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_AmericanIndianOrAlaskaNativeAlone + source_sv_ids: + # Population: Female, 100,000 USD or More, American Indian or Alaska Native Alone, Not Worked Full Time + - dc/bdl1bpzj3klq4 + # Population: Female, 100,000 USD or More, American Indian or Alaska Native Alone, Worked Full Time + - dc/8v29dc0442ljg + # Population: Male, 100,000 USD or More, American Indian or Alaska Native Alone, Not Worked Full Time + - dc/4130fr4llf992 + # Population: Male, 100,000 USD or More, American Indian or Alaska Native Alone, Worked Full Time + - dc/ehgx6y0qyncl2 + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_AsianAlone + source_sv_ids: + # Population: Female, 100,000 USD or More, Asian Alone, Not Worked Full Time + - dc/5zrn0w3nwe9e2 + # Population: Female, 100,000 USD or More, Asian Alone, Worked Full Time + - dc/sk4nqnbdskvhg + # Population: Male, 100,000 USD or More, Asian Alone, Not Worked Full Time + - dc/w4xl5kcbyjmnb + # Population: Male, 100,000 USD or More, Asian Alone, Worked Full Time + - dc/68zc6qjdtxllf + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_BlackOrAfricanAmericanAlone + source_sv_ids: + # Population: Female, 100,000 USD or More, Black or African American Alone, Not Worked Full Time + - dc/hndxvvnzf2yr1 + # Population: Female, 100,000 USD or More, Black or African American Alone, Worked Full Time + - dc/vf30v7c1t5lz5 + # Population: Male, 100,000 USD or More, Black or African American Alone, Not Worked Full Time + - dc/nm3bftsv2z2w2 + # Population: Male, 100,000 USD or More, Black or African American Alone, Worked Full Time + - dc/whevye7y9vg1c + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_HispanicOrLatino + source_sv_ids: + # Population: Female, 100,000 USD or More, Hispanic or Latino, Not Worked Full Time + - dc/nwzb4v1hp2jw4 + # Population: Female, 100,000 USD or More, Hispanic or Latino, Worked Full Time + - dc/w5geg8s1lmb04 + # Population: Male, 100,000 USD or More, Hispanic or Latino, Not Worked Full Time + - dc/xdrmxlpjlcrrb + # Population: Male, 100,000 USD or More, Hispanic or Latino, Worked Full Time + - dc/espye40zdmhgg + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_NativeHawaiianOrOtherPacificIslanderAlone + source_sv_ids: + # Population: Female, 100,000 USD or More, Native Hawaiian or Other Pacific Islander Alone, Not Worked Full Time + - dc/8bwnnflm9kwbg + # Population: Female, 100,000 USD or More, Native Hawaiian or Other Pacific Islander Alone, Worked Full Time + - dc/4r5pvjszn5x03 + # Population: Male, 100,000 USD or More, Native Hawaiian or Other Pacific Islander Alone, Not Worked Full Time + - dc/mx7edmkkrxvl9 + # Population: Male, 100,000 USD or More, Native Hawaiian or Other Pacific Islander Alone, Worked Full Time + - dc/ce7frpwyfpx85 + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_SomeOtherRaceAlone + source_sv_ids: + # Population: Female, 100,000 USD or More, Some Other Race Alone, Not Worked Full Time + - dc/y70ppl4rxjhjh + # Population: Female, 100,000 USD or More, Some Other Race Alone, Worked Full Time + - dc/3zq436nrf2d66 + # Population: Male, 100,000 USD or More, Some Other Race Alone, Not Worked Full Time + - dc/pgrfnkfnswr5d + # Population: Male, 100,000 USD or More, Some Other Race Alone, Worked Full Time + - dc/cmnkc5fpvpfk9 + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_TwoOrMoreRaces + source_sv_ids: + # Population: Female, 100,000 USD or More, Two or More Races, Not Worked Full Time + - dc/8lffdmtgqv269 + # Population: Female, 100,000 USD or More, Two or More Races, Worked Full Time + - dc/n4z306vm3zdm6 + # Population: Male, 100,000 USD or More, Two or More Races, Not Worked Full Time + - dc/yr2g2ct310d43 + # Population: Male, 100,000 USD or More, Two or More Races, Worked Full Time + - dc/em5ysxsf96xb8 + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_WhiteAlone + source_sv_ids: + # Population: Female, 100,000 USD or More, White Alone, Not Worked Full Time + - dc/kd7qcdzs2q8p4 + # Population: Female, 100,000 USD or More, White Alone, Worked Full Time + - dc/gjkbs2qeb9m0g + # Population: Male, 100,000 USD or More, White Alone, Not Worked Full Time + - dc/dhjfv19drrr9 + # Population: Male, 100,000 USD or More, White Alone, Worked Full Time + - dc/z76nk40e6qtn8 + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar100000Onwards_WithIncome_WhiteAloneNotHispanicOrLatino + source_sv_ids: + # Population: Female, 100,000 USD or More, White Alone Not Hispanic or Latino, Not Worked Full Time + - dc/elr21hp41d4jh + # Population: Female, 100,000 USD or More, White Alone Not Hispanic or Latino, Worked Full Time + - dc/cqc4zh1dg8s6d + # Population: Male, 100,000 USD or More, White Alone Not Hispanic or Latino, Not Worked Full Time + - dc/7g2mm704kfq63 + # Population: Male, 100,000 USD or More, White Alone Not Hispanic or Latino, Worked Full Time + - dc/ekh1g39v9sgj4 + + # The following 9 aggregations are: Low income (10,000 - 12,499 USD), by race. + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_AmericanIndianOrAlaskaNativeAlone + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, American Indian or Alaska Native Alone, Not Worked Full Time + - dc/g1lqz36qc6x26 + # Population: Female, 10,000 - 12,499 USD, American Indian or Alaska Native Alone, Worked Full Time + - dc/nfj1c6f5xvjpf + # Population: Male, 10,000 - 12,499 USD, American Indian or Alaska Native Alone, Not Worked Full Time + - dc/1cytg38zh4kmb + # Population: Male, 10,000 - 12,499 USD, American Indian or Alaska Native Alone, Worked Full Time + - dc/kvpkg8xd66q1d + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_AsianAlone + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, Asian Alone, Not Worked Full Time + - dc/rkylr8pzrhvrb + # Population: Female, 10,000 - 12,499 USD, Asian Alone, Worked Full Time + - dc/ecz488knjvddf + # Population: Male, 10,000 - 12,499 USD, Asian Alone, Not Worked Full Time + - dc/l481yty608g4 + # Population: Male, 10,000 - 12,499 USD, Asian Alone, Worked Full Time + - dc/f6psb62vxhcgb + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_BlackOrAfricanAmericanAlone + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, Black or African American Alone, Not Worked Full Time + - dc/zvn4y702wsxe3 + # Population: Female, 10,000 - 12,499 USD, Black or African American Alone, Worked Full Time + - dc/6jgshc72wscd7 + # Population: Male, 10,000 - 12,499 USD, Black or African American Alone, Not Worked Full Time + - dc/h255gx042f7y6 + # Population: Male, 10,000 - 12,499 USD, Black or African American Alone, Worked Full Time + - dc/pd0td6p7wbgvc + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_HispanicOrLatino + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, Hispanic or Latino, Not Worked Full Time + - dc/p5ewnb0xtrrp + # Population: Female, 10,000 - 12,499 USD, Hispanic or Latino, Worked Full Time + - dc/7ev9gs9ppxbq2 + # Population: Male, 10,000 - 12,499 USD, Hispanic or Latino, Not Worked Full Time + - dc/t5d500j7754qb + # Population: Male, 10,000 - 12,499 USD, Hispanic or Latino, Worked Full Time + - dc/x2p3338cck7vg + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_NativeHawaiianOrOtherPacificIslanderAlone + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, Native Hawaiian or Other Pacific Islander Alone, Not Worked Full Time + - dc/xbq05lkgsx8b6 + # Population: Female, 10,000 - 12,499 USD, Native Hawaiian or Other Pacific Islander Alone, Worked Full Time + - dc/nxqjjpwkkxpx4 + # Population: Male, 10,000 - 12,499 USD, Native Hawaiian or Other Pacific Islander Alone, Not Worked Full Time + - dc/bd5nldln2dxng + # Population: Male, 10,000 - 12,499 USD, Native Hawaiian or Other Pacific Islander Alone, Worked Full Time + - dc/rgme2je6kbj27 + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_SomeOtherRaceAlone + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, Some Other Race Alone, Not Worked Full Time + - dc/8ws12v06y0sqh + # Population: Female, 10,000 - 12,499 USD, Some Other Race Alone, Worked Full Time + - dc/43pxcsbgjksmg + # Population: Male, 10,000 - 12,499 USD, Some Other Race Alone, Not Worked Full Time + - dc/lj0q1zj5d8df + # Population: Male, 10,000 - 12,499 USD, Some Other Race Alone, Worked Full Time + - dc/sy61mbcgrsz73 + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_TwoOrMoreRaces + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, Two or More Races, Not Worked Full Time + - dc/see1v777j4807 + # Population: Female, 10,000 - 12,499 USD, Two or More Races, Worked Full Time + - dc/422zpgm23b2m7 + # Population: Male, 10,000 - 12,499 USD, Two or More Races, Not Worked Full Time + - dc/6k4k59zwbgw92 + # Population: Male, 10,000 - 12,499 USD, Two or More Races, Worked Full Time + - dc/qdrpgr3zby8k + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_WhiteAlone + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, White Alone, Not Worked Full Time + - dc/c42c6rmszjdv7 + # Population: Female, 10,000 - 12,499 USD, White Alone, Worked Full Time + - dc/bpzm3y2rxt7y9 + # Population: Male, 10,000 - 12,499 USD, White Alone, Not Worked Full Time + - dc/kv7x6r2cqyxcf + # Population: Male, 10,000 - 12,499 USD, White Alone, Worked Full Time + - dc/xzg5t6eq4pnph + - ancestor_sv_id: Count_Person_16OrMoreYears_USDollar10000To12499_WithIncome_WhiteAloneNotHispanicOrLatino + source_sv_ids: + # Population: Female, 10,000 - 12,499 USD, White Alone Not Hispanic or Latino, Not Worked Full Time + - dc/t6b45v6bsdf1 + # Population: Female, 10,000 - 12,499 USD, White Alone Not Hispanic or Latino, Worked Full Time + - dc/j8dgk3864jpbd + # Population: Male, 10,000 - 12,499 USD, White Alone Not Hispanic or Latino, Not Worked Full Time + - dc/bv5sd8hrhs82h + # Population: Male, 10,000 - 12,499 USD, White Alone Not Hispanic or Latino, Worked Full Time + - dc/5jp07brw3g26h + + - name: "DEA_ARCOS: Opioid Prescription SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - IndiaNSS_HealthAilments + output_import: IndiaNSS_HealthAilments_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionBloodDisease_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionBloodDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionBloodDisease_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionCancer_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionCancer_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionCancer_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionCardiovascularDisease_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionEarCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionEarCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionEarCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionEndocrineMetabolicCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionEyeCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionEyeCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionEyeCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionGastroIntestinalCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionGenitoUrinaryCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionInfection_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionInfection_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionInfection_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionInjury_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionInjury_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionInjury_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionMusculoSkeletalCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionNSSHealthOtherCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionObstetricCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionObstetricCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionObstetricCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionPsychiatricNeurologicalCondition_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionRespiratoryDisease_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + - ancestor_sv_id: Count_MedicalConditionIncident_ConditionSkinDisease_AsAFractionOf_Count_Person + source_sv_ids: + - Count_MedicalConditionIncident_0To4Years_Female_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Female_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Female_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Female_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Female_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Female_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Female_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Female_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_0To4Years_Male_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_15To29Years_Male_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_30To44Years_Male_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_45To59Years_Male_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_5To14Years_Male_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60OrMoreYears_Male_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_60To69Years_Male_ConditionSkinDisease_AsAFractionOf_Count_Person + - Count_MedicalConditionIncident_70OrMoreYears_Male_ConditionSkinDisease_AsAFractionOf_Count_Person + skip_all_sources_present_check: true + + - name: "OECDRegionalDemography: Person Age Groups SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - OECDRegionalDemography_Population + output_import: OECDRegionalDemography_Person_Age_StatVarAgg + stat_var_aggregation: + aggregations: + # OECD, Age group + - ancestor_sv_id: Count_Person_Upto9Years + source_sv_ids: + - Count_Person_Upto4Years + - Count_Person_5To9Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_10To19Years + source_sv_ids: + - Count_Person_10To14Years + - Count_Person_15To19Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_20To29Years + source_sv_ids: + - Count_Person_20To24Years + - Count_Person_25To29Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_30To39Years + source_sv_ids: + - Count_Person_30To34Years + - Count_Person_35To39Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_40To49Years + source_sv_ids: + - Count_Person_40To44Years + - Count_Person_45To49Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_50To59Years + source_sv_ids: + - Count_Person_50To54Years + - Count_Person_55To59Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_60To69Years + source_sv_ids: + - Count_Person_60To64Years + - Count_Person_65To69Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_70OrMoreYears + source_sv_ids: + - Count_Person_70To74Years + - Count_Person_75To79Years + - Count_Person_80OrMoreYears + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_5To19Years + source_sv_ids: + - Count_Person_5To9Years + - Count_Person_10To14Years + - Count_Person_15To19Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_25To34Years + source_sv_ids: + - Count_Person_25To29Years + - Count_Person_30To34Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_35To44Years + source_sv_ids: + - Count_Person_35To39Years + - Count_Person_40To44Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_45To54Years + source_sv_ids: + - Count_Person_45To49Years + - Count_Person_50To54Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_55To64Years + source_sv_ids: + - Count_Person_55To59Years + - Count_Person_60To64Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_65To74Years + source_sv_ids: + - Count_Person_65To69Years + - Count_Person_70To74Years + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_75OrMoreYears + source_sv_ids: + - Count_Person_75To79Years + - Count_Person_80OrMoreYears + skip_all_sources_present_check: true + + - name: "CensusACS5YearSurvey_SubjectTables_S0801: Work Commute SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey_SubjectTables_S0801 + output_import: CensusACS5YearSurvey_SubjectTables_S0801_StatVarAgg + stat_var_aggregation: + aggregations: + # Census ACS 5 year, Work Commute group + - ancestor_sv_id: Count_Person_Years16Onwards_WorkCommute_Employed + source_sv_ids: + - Count_Person_Years16Onwards_CarTruckOrVan_WorkCommute_Employed + - Count_Person_Years16Onwards_DroveAlone_WorkCommute_Employed + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_WorkCommute_Employed + - Count_Person_Years16Onwards_CarpoolOccupancy2Person_CarTruckOrVanCarpooled_WorkCommute_Employed + - Count_Person_Years16Onwards_CarpoolOccupancy3Person_CarTruckOrVanCarpooled_WorkCommute_Employed + - Count_Person_Years16Onwards_CarpoolOccupancy4OrMorePerson_CarTruckOrVanCarpooled_WorkCommute_Employed + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_WorkCommute_Employed + - Count_Person_Years16Onwards_Walked_WorkCommute_Employed + - Count_Person_Years16Onwards_Bicycle_WorkCommute_Employed + - Count_Person_Years16Onwards_TaxicabMotorcycleBicycleOrOtherMeans_WorkCommute_Employed + - Count_Person_Years16Onwards_WorkedAtHome_WorkCommute_Employed + - Count_Person_Years16Onwards_WorkedFromHome_WorkCommute_Employed + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_Years16Onwards_WorkCommute_Employed_Male + source_sv_ids: + - Count_Person_Years16Onwards_CarTruckOrVan_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_DroveAlone_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_CarpoolOccupancy2Person_CarTruckOrVanCarpooled_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_CarpoolOccupancy3Person_CarTruckOrVanCarpooled_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_CarpoolOccupancy4OrMorePerson_CarTruckOrVanCarpooled_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_Walked_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_Bicycle_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_TaxicabMotorcycleBicycleOrOtherMeans_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_WorkedAtHome_WorkCommute_Employed_Male + - Count_Person_Years16Onwards_WorkedFromHome_WorkCommute_Employed_Male + skip_all_sources_present_check: true + - ancestor_sv_id: Count_Person_Years16Onwards_WorkCommute_Employed_Female + source_sv_ids: + - Count_Person_Years16Onwards_CarTruckOrVan_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_DroveAlone_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_CarpoolOccupancy2Person_CarTruckOrVanCarpooled_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_CarpoolOccupancy3Person_CarTruckOrVanCarpooled_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_CarpoolOccupancy4OrMorePerson_CarTruckOrVanCarpooled_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_Walked_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_Bicycle_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_TaxicabMotorcycleBicycleOrOtherMeans_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_WorkedAtHome_WorkCommute_Employed_Female + - Count_Person_Years16Onwards_WorkedFromHome_WorkCommute_Employed_Female + skip_all_sources_present_check: true + # Worked outside of Home + - ancestor_sv_id: Count_Person_Years16Onwards_WorkCommute_Employed_WorkedOutsideOfHome + source_sv_ids: + - Count_Person_Years16Onwards_StartTimeHour0000To0459_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0500To0529_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0530To0559_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0600To0629_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0630To0659_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0700To0729_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0730To0759_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0800To0829_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0830To0859_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0900To2359_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_10OrLessMinute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_10To14Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_15To19Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_20To24Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_25To29Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_30To34Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_35To44Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_45To59Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_60OrMoreMinute_WorkCommute_Employed_WorkedOutsideOfHome + - ancestor_sv_id: Count_Person_Years16Onwards_WorkCommute_Employed_Male_WorkedOutsideOfHome + source_sv_ids: + - Count_Person_Years16Onwards_StartTimeHour0000To0459_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0500To0529_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0530To0559_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0600To0629_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0630To0659_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0700To0729_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0730To0759_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0800To0829_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0830To0859_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0900To2359_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_10OrLessMinute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_10To14Minute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_15To19Minute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_20To24Minute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_25To29Minute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_30To34Minute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_35To44Minute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_45To59Minute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - Count_Person_Years16Onwards_60OrMoreMinute_WorkCommute_Employed_Male_WorkedOutsideOfHome + - ancestor_sv_id: Count_Person_Years16Onwards_WorkCommute_Employed_Female_WorkedOutsideOfHome + source_sv_ids: + - Count_Person_Years16Onwards_StartTimeHour0000To0459_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0500To0529_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0530To0559_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0600To0629_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0630To0659_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0700To0729_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0730To0759_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0800To0829_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0830To0859_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_StartTimeHour0900To2359_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_10OrLessMinute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_10To14Minute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_15To19Minute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_20To24Minute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_25To29Minute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_30To34Minute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_35To44Minute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_45To59Minute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - Count_Person_Years16Onwards_60OrMoreMinute_WorkCommute_Employed_Female_WorkedOutsideOfHome + - name: "CensusACS5YearSurvey_SubjectTables_S0804: Means of Transportation Work Commute SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - CensusACS5YearSurvey_SubjectTables_S0804 + output_import: CensusACS5YearSurvey_SubjectTables_S0804_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_WorkCommute_Employed_WorkedOutsideOfHome + source_sv_ids: + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_10OrLessMinute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_10To14Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_15To19Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_20To24Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_25To29Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_30To34Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_35To44Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_45To59Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_PublicTransportationExcludingTaxicab_60OrMoreMinute_WorkCommute_Employed_WorkedOutsideOfHome + - ancestor_sv_id: Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_WorkCommute_Employed_WorkedOutsideOfHome + source_sv_ids: + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_10OrLessMinute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_10To14Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_15To19Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_20To24Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_25To29Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_30To34Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_35To44Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_45To59Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanDroveAlone_60OrMoreMinute_WorkCommute_Employed_WorkedOutsideOfHome + - ancestor_sv_id: Count_Person_Years16Onwards_CarTruckOrVanCarpooled_WorkCommute_Employed_WorkedOutsideOfHome + source_sv_ids: + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_10OrLessMinute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_10To14Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_15To19Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_25To29Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_20To24Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_30To34Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_35To44Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_45To59Minute_WorkCommute_Employed_WorkedOutsideOfHome + - Count_Person_Years16Onwards_CarTruckOrVanCarpooled_60OrMoreMinute_WorkCommute_Employed_WorkedOutsideOfHome + - name: "India_RBIStateDomesticProduct: Gross Value Added SV Aggregation" + type: STAT_VAR_AGGREGATION + input_imports: + - India_RBIStateDomesticProduct + output_import: India_RBIStateDomesticProduct_StatVarAgg + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Nominal_Amount_EconomicActivity_GrossValueAdded + source_sv_ids: + - Nominal_Amount_EconomicActivity_GrossValueAdded_Agriculture + - Nominal_Amount_EconomicActivity_GrossValueAdded_BankingAndInsuranceSector + - Nominal_Amount_EconomicActivity_GrossValueAdded_Construction + - Nominal_Amount_EconomicActivity_GrossValueAdded_Industry + - Nominal_Amount_EconomicActivity_GrossValueAdded_ManufacturingSector + - Nominal_Amount_EconomicActivity_GrossValueAdded_Services + - ancestor_sv_id: Nominal_Amount_EconomicActivity_NetValueAdded + source_sv_ids: + - Nominal_Amount_EconomicActivity_NetValueAdded_Agriculture + - Nominal_Amount_EconomicActivity_NetValueAdded_Construction + - Nominal_Amount_EconomicActivity_NetValueAdded_Industry + - Nominal_Amount_EconomicActivity_NetValueAdded_ManufacturingSector + - Nominal_Amount_EconomicActivity_NetValueAdded_Services + - ancestor_sv_id: Amount_EconomicActivity_GrossValueAdded_RealValue + source_sv_ids: + - RealValue_Amount_EconomicActivity_GrossValueAdded_Agriculture + - RealValue_Amount_EconomicActivity_GrossValueAdded_BankingAndInsuranceSector + - RealValue_Amount_EconomicActivity_GrossValueAdded_Construction + - RealValue_Amount_EconomicActivity_GrossValueAdded_Industry + - RealValue_Amount_EconomicActivity_GrossValueAdded_ManufacturingSector + - RealValue_Amount_EconomicActivity_GrossValueAdded_Services + - ancestor_sv_id: RealValue_Amount_EconomicActivity_NetValueAdded + source_sv_ids: + - RealValue_Amount_EconomicActivity_NetValueAdded_Agriculture + - RealValue_Amount_EconomicActivity_NetValueAdded_Construction + - RealValue_Amount_EconomicActivity_NetValueAdded_Industry + - RealValue_Amount_EconomicActivity_NetValueAdded_ManufacturingSector + - RealValue_Amount_EconomicActivity_NetValueAdded_Services diff --git a/pipeline/workflow/ingestion-helper/aggregation/configs/statvar_calculation.yaml b/pipeline/workflow/ingestion-helper/aggregation/configs/statvar_calculation.yaml new file mode 100644 index 000000000..4969e1a98 --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/configs/statvar_calculation.yaml @@ -0,0 +1,315 @@ +calculations: + + # Energy. + - name: "EIA_Electricity: Annual Emissions Per Capita StatVar Calculation" + type: STAT_VAR_CALCULATION + input_imports: + - EPA_GHGRP_AggCounty + - EPA_GHGRP_AggCensusZipCodeTabulationArea + - EPA_GHGRP_AggCounty_AggState + - EIA_Electricity + - UNEnergy + output_import: Energy_StatVarCalculation + stat_var_calculation: + calculations: + - input1: + sv_regex: Annual_Emissions_GreenhouseGas_NonBiogenic + measurement_method_regex: dcAggregate/EPA_GHGRP + facet_info: + unit: MetricTonCO2e + observation_period: P1Y + input2: + sv_regex: Annual_Generation_Electricity + facet_info: + unit: GigawattHour + operation: DIVIDE + output: + sv: Annual_Emissions_GreenhouseGas_NonBiogenic_Per_Annual_Generation_Electricity + measurement_method: EPA_GHGRP_EIA_Electricity + facet_info: + unit: MetricTonCO2ePerGigawattHour + observation_period: P1Y + + # Climate: Temperature modeling diffs from actual. + # There are 30+ models. For each model, the output diffs are done for + # Aggregations: Mean, Min, Max and also for SSP2 with RCP4.5 and SSP5 with + # RCP8.5 (for each model). + # This includes diffs for P1M, P1Y, P5Y and P10Y. + # Note that the computations below use sv_regex and measurement_method_regex + # instead of sv and measurement_method to allow more expressive matching. Also + # note that the output uses sv_prefix and measurement_method_prefix which is not + # the case above, for example. Whenever sv and measurement_method are set in the + # output, they are preferred. Otherwise, the *_prefix is used. + - name: "Climate: NASA CMIP6 Temperature Modeling Diffs Calculation" + type: STAT_VAR_CALCULATION + input_imports: + - NASA_NEXGDDP_CMIP6_Subnational + - NASA_NEXGDDP_CMIP6_IpccPlaces50 + - NOAA_EPA_Observed_Historical_Weather + - NASA_NEXDCP30 + - Copernicus_ECMWF_ERA5_Monthly + - Copernicus_ECMWF_ERA5_Monthly_AggrYearsStats + - NASA_NEXDCP30_AggrYearsStats + - NASA_NEXGDDP_CMIP6_IpccPlaces50_AggrYearsStats + - NASA_NEXGDDP_CMIP6_Subnational_AggrYearsStats + - NOAA_EPA_Observed_Historical_Weather_AggrYearsStats + output_import: Climate_StatVarCalculation + stat_var_calculation: + calculations: + - input1: + sv_regex: "^Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P1M + input2: + sv_regex: "^Mean_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P1M + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P1M + + - input1: + sv_regex: "^Min_Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P1M + input2: + sv_regex: "^Min_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P1M + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P1M + + - input1: + sv_regex: "^Max_Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P1M + input2: + sv_regex: "^Max_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P1M + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P1M + + - input1: + sv_regex: "^Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P1Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Mean_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P1Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P1Y + + - input1: + sv_regex: "^Min_Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P1Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Min_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P1Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P1Y + + - input1: + sv_regex: "^Max_Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P1Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Max_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P1Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P1Y + + - input1: + sv_regex: "^Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P5Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Mean_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P5Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P5Y + + - input1: + sv_regex: "^Min_Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P5Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Min_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P5Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P5Y + + - input1: + sv_regex: "^Max_Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P5Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Max_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P5Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P5Y + + - input1: + sv_regex: "^Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P10Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Mean_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P10Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P10Y + + - input1: + sv_regex: "^Min_Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P10Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Min_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P10Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P10Y + + - input1: + sv_regex: "^Max_Temperature(_SSP[0-9]+)*$" + measurement_method_regex: "^dcAggregate/NASA_Mean_CMIP6_.*" + facet_info: + unit: Celsius + observation_period: P10Y + import_name_regex: ".*AggrYearsStats$" + input2: + sv_regex: "^Max_Temperature$" + measurement_method_regex: "dcAggregate/NASAGSOD_NASAGHCN_EPA" + facet_info: + unit: Celsius + observation_period: P10Y + import_name_regex: ".*AggrYearsStats$" + operation: SUBTRACT + output: + sv_prefix: "DifferenceRelativeToObservationalData_" + measurement_method_prefix: "dcAggregate/NASA_Mean_CMIP6_WithBaseAs_" + facet_info: + unit: Celsius + observation_period: P10Y diff --git a/pipeline/workflow/ingestion-helper/aggregation/configs/statvar_series.yaml b/pipeline/workflow/ingestion-helper/aggregation/configs/statvar_series.yaml new file mode 100644 index 000000000..0d8b18280 --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/configs/statvar_series.yaml @@ -0,0 +1,336 @@ +calculations: + - name: "NASA_NEXDCP30: Measurement Methods & Base Date Diff Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 1 + input_imports: + - NASA_NEXDCP30 + output_import: NASA_NEXDCP30_AggrDiffStats + stat_var_series_aggregation: + aggr_funcs: + - max_diff_across_measurement_methods: {} + - diff_relative_to_base_date: + date_specs: + - dates: + - "1990" + - "2006" + + - name: "NASA_NEXGDDP_Subnational: Measurement Methods & Base Date Diff Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 1 + input_imports: + - NASA_NEXGDDP_Subnational + output_import: NASA_NEXGDDP_Subnational_AggrDiffStats + stat_var_series_aggregation: + aggr_funcs: + - max_diff_across_measurement_methods: {} + - diff_relative_to_base_date: + date_specs: + - start_date: "2006" + end_date: "2020" + - dates: + - "1990" + - "2006" + + - name: "NASA_NEXGDDP_Country: Measurement Methods & Base Date Diff Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 1 + input_imports: + - NASA_NEXGDDP_Country + output_import: NASA_NEXGDDP_Country_AggrDiffStats + stat_var_series_aggregation: + aggr_funcs: + - max_diff_across_measurement_methods: {} + - diff_relative_to_base_date: + date_specs: + - dates: + - "1990" + - "2006" + + - name: "NASA_NEXGDDP_CMIP6_Subnational: Measurement Methods & Base Date Diff Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 1 + input_imports: + - NASA_NEXGDDP_CMIP6_Subnational + output_import: NASA_NEXGDDP_CMIP6_Subnational_AggrDiffStats + stat_var_series_aggregation: + aggr_funcs: + - max_diff_across_measurement_methods: {} + - diff_relative_to_base_date: + date_specs: + - start_date: "2015" + end_date: "2020" + - dates: + - "1990" + - "2006" + + - name: "NASA_NEXGDDP_CMIP6_IpccPlaces50: Measurement Methods & Base Date Diff Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 1 + input_imports: + - NASA_NEXGDDP_CMIP6_IpccPlaces50 + output_import: NASA_NEXGDDP_CMIP6_IpccPlaces50_AggrDiffStats + stat_var_series_aggregation: + aggr_funcs: + - max_diff_across_measurement_methods: {} + - diff_relative_to_base_date: + date_specs: + - start_date: "2015" + end_date: "2020" + - dates: + - "1990" + - "2006" + + - name: "NASA_NEXGDDP_CMIP6_Subnational: Stats Across Models Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 2 + input_imports: + - NASA_NEXGDDP_CMIP6_Subnational + output_import: NASA_NEXGDDP_CMIP6_Subnational_AggrStatsAcrossModels + stat_var_series_aggregation: + aggr_funcs: + - aggr_stats_across_models: + sv_regex: "^DifferenceRelativeToObservationalData_.*" + aggregation_ops: + - OPERATOR_MEDIAN + - OPERATOR_PERCENTILE90 + - OPERATOR_PERCENTILE10 + + - name: "NASA_NEXGDDP_CMIP6_IpccPlaces50: Stats Across Models Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 2 + input_imports: + - NASA_NEXGDDP_CMIP6_IpccPlaces50 + output_import: NASA_NEXGDDP_CMIP6_IpccPlaces50_AggrStatsAcrossModels + stat_var_series_aggregation: + aggr_funcs: + - aggr_stats_across_models: + sv_regex: "^DifferenceRelativeToObservationalData_.*" + aggregation_ops: + - OPERATOR_MEDIAN + - OPERATOR_PERCENTILE90 + - OPERATOR_PERCENTILE10 + + - name: "NASA_NEXGDDP_CMIP6_Subnational: Over Time Stats Across Models Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 3 + input_imports: + - NASA_NEXGDDP_CMIP6_Subnational_AggrStatsAcrossModels + output_import: NASA_NEXGDDP_CMIP6_Subnational_AggrOverTimeOnStatsAcrossModels + stat_var_series_aggregation: + aggr_funcs: + # Config for future 10-year max/min aggregate + - aggr_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P10Y + output_obs_date: "2030" + sv_configs: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + aggregation_op: OPERATOR_MAX + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + aggregation_op: OPERATOR_MIN + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate((?!Max|Min).)*_Temperature.*" + aggregation_op: OPERATOR_MEAN + + # Config for future 30-year max/min aggregate + - aggr_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P30Y + output_obs_date: "2050" + sv_configs: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + aggregation_op: OPERATOR_MAX + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + aggregation_op: OPERATOR_MIN + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate((?!Max|Min).)*_Temperature.*" + aggregation_op: OPERATOR_MEAN + + # Config for future 80-year max/min aggregate + - aggr_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P80Y + output_obs_date: "2100" + sv_configs: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + aggregation_op: OPERATOR_MAX + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + aggregation_op: OPERATOR_MIN + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate((?!Max|Min).)*_Temperature.*" + aggregation_op: OPERATOR_MEAN + + # Config for future 10-year period of num-months with max/min temp above/below threshold + - count_threshold_exception_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P10Y + output_obs_date: "2030" + thresholds: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + threshold_value: 5 + unit: Celsius + comparison: OPERATOR_GE + output_cprop: maxTemperature + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + threshold_value: -5 + unit: Celsius + comparison: OPERATOR_LE + output_cprop: minTemperature + + # Config for future 30-year period of num-months with max/min temp above/below threshold + - count_threshold_exception_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P30Y + output_obs_date: "2050" + thresholds: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + threshold_value: 5 + unit: Celsius + comparison: OPERATOR_GE + output_cprop: maxTemperature + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + threshold_value: -5 + unit: Celsius + comparison: OPERATOR_LE + output_cprop: minTemperature + + # Config for future 80-year period of num-months with max/min temp above/below threshold + - count_threshold_exception_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P80Y + output_obs_date: "2100" + thresholds: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + threshold_value: 5 + unit: Celsius + comparison: OPERATOR_GE + output_cprop: maxTemperature + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + threshold_value: -5 + unit: Celsius + comparison: OPERATOR_LE + output_cprop: minTemperature + + - name: "NASA_NEXGDDP_CMIP6_IpccPlaces50: Over Time Stats Across Models Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 3 + input_imports: + - NASA_NEXGDDP_CMIP6_IpccPlaces50_AggrStatsAcrossModels + output_import: NASA_NEXGDDP_CMIP6_IpccPlaces50_AggrOverTimeOnStatsAcrossModels + stat_var_series_aggregation: + aggr_funcs: + # Config for future 10-year max/min aggregate + - aggr_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P10Y + output_obs_date: "2030" + sv_configs: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + aggregation_op: OPERATOR_MAX + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + aggregation_op: OPERATOR_MIN + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate((?!Max|Min).)*_Temperature.*" + aggregation_op: OPERATOR_MEAN + + # Config for future 30-year max/min aggregate + - aggr_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P30Y + output_obs_date: "2050" + sv_configs: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + aggregation_op: OPERATOR_MAX + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + aggregation_op: OPERATOR_MIN + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate((?!Max|Min).)*_Temperature.*" + aggregation_op: OPERATOR_MEAN + + # Config for future 80-year max/min aggregate + - aggr_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P80Y + output_obs_date: "2100" + sv_configs: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + aggregation_op: OPERATOR_MAX + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + aggregation_op: OPERATOR_MIN + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate((?!Max|Min).)*_Temperature.*" + aggregation_op: OPERATOR_MEAN + + # Config for future 10-year period of num-months with max/min temp above/below threshold + - count_threshold_exception_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P10Y + output_obs_date: "2030" + thresholds: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + threshold_value: 5 + unit: Celsius + comparison: OPERATOR_GE + output_cprop: maxTemperature + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + threshold_value: -5 + unit: Celsius + comparison: OPERATOR_LE + output_cprop: minTemperature + + # Config for future 30-year period of num-months with max/min temp above/below threshold + - count_threshold_exception_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P30Y + output_obs_date: "2050" + thresholds: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + threshold_value: 5 + unit: Celsius + comparison: OPERATOR_GE + output_cprop: maxTemperature + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + threshold_value: -5 + unit: Celsius + comparison: OPERATOR_LE + output_cprop: minTemperature + + # Config for future 80-year period of num-months with max/min temp above/below threshold + - count_threshold_exception_over_time: + time_range: + input_obs_period: P1M + output_obs_period: P80Y + output_obs_date: "2100" + thresholds: + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Max_Temperature.*" + threshold_value: 5 + unit: Celsius + comparison: OPERATOR_GE + output_cprop: maxTemperature + - sv_regex: "^(Median|Percentile90|Percentile10)AcrossModels_DifferenceRelativeToBaseDate.*_Min_Temperature.*" + threshold_value: -5 + unit: Celsius + comparison: OPERATOR_LE + output_cprop: minTemperature + + - name: "NASA_NEXGDDP_IpccPlaces50: Measurement Methods & Base Date Diff Series Aggregation" + type: STAT_VAR_SERIES_AGGREGATION + stage: 1 + input_imports: + - NASA_NEXGDDP_IpccPlaces50 + output_import: NASA_NEXGDDP_IpccPlaces50_AggrDiffStats + stat_var_series_aggregation: + aggr_funcs: + - max_diff_across_measurement_methods: {} + - diff_relative_to_base_date: + date_specs: + - start_date: "2015" + end_date: "2020" + - dates: + - "1990" + - "2006" diff --git a/pipeline/workflow/ingestion-helper/aggregation/configs/super_enum.yaml b/pipeline/workflow/ingestion-helper/aggregation/configs/super_enum.yaml new file mode 100644 index 000000000..cd62f1877 --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/configs/super_enum.yaml @@ -0,0 +1,30 @@ +calculations: + - name: "CensusACS5YearSurvey: Super Enum Mapping Aggregation" + type: SUPER_ENUM_AGGREGATION + input_imports: + - CensusACS5YearSurvey + output_import: CensusACS5YearSurvey_SuperEnum + + - name: "CensusACS5YearSurvey_AggCountry: Super Enum Mapping Aggregation" + type: SUPER_ENUM_AGGREGATION + input_imports: + - CensusACS5YearSurvey_AggCountry + output_import: CensusACS5YearSurvey_AggCountry_SuperEnum + + - name: "NCES_PrivateSchoolStats: Super Enum Mapping Aggregation" + type: SUPER_ENUM_AGGREGATION + input_imports: + - NCES_PrivateSchoolStats + output_import: NCES_PrivateSchoolStats_SuperEnum + + - name: "NCES_PublicSchoolStats: Super Enum Mapping Aggregation" + type: SUPER_ENUM_AGGREGATION + input_imports: + - NCES_PublicSchoolStats + output_import: NCES_PublicSchoolStats_SuperEnum + + - name: "NCES_SchoolDistrictStats: Super Enum Mapping Aggregation" + type: SUPER_ENUM_AGGREGATION + input_imports: + - NCES_SchoolDistrictStats + output_import: NCES_SchoolDistrictStats_SuperEnum diff --git a/pipeline/workflow/ingestion-helper/aggregation/orchestrator.py b/pipeline/workflow/ingestion-helper/aggregation/orchestrator.py new file mode 100644 index 000000000..bf42d2d53 --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/orchestrator.py @@ -0,0 +1,316 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Aggregation orchestrator for Data Commons ingestion workflow.""" + +import glob +import logging +import os +import time +from typing import Any, Dict, List, Optional + +from .bq_executor import BigQueryExecutor +from .linked_edge_generator import LinkedEdgeGenerator +from .place_aggregation_generator import PlaceAggregationGenerator +from .provenance_summary_generator import ProvenanceSummaryGenerator +from .stat_var_aggregator import StatVarAggregator +from .stat_var_calculation_generator import StatVarCalculationGenerator +from .stat_var_group_generator import StatVarGroupGenerator +from .validator import validate_config + + +class AggregationOrchestrator: + """Orchestrates the overall aggregation workflow across multi-stage execution.""" + + def __init__( + self, + connection_id: str, + project_id: str, + instance_id: str, + database_id: str, + location: Optional[str] = None, + is_base_dc: bool = True, + config_dir: Optional[str] = None, + config_file_path: Optional[str] = None + ) -> None: + """Initializes the orchestrator and loads/validates configuration files. + + Args: + connection_id: BigQuery connection ID to Spanner. + project_id: GCP Project ID. + instance_id: Spanner Instance ID. + database_id: Spanner Database ID. + location: BigQuery location. + is_base_dc: Whether this is running in the base Data Commons environment. + config_dir: Directory containing aggregation YAML configs (default: configs/). + config_file_path: Optional path to single config file or directory. + """ + self.executor = BigQueryExecutor( + connection_id=connection_id, + project_id=project_id, + instance_id=instance_id, + database_id=database_id, + location=location, + run_sequential=False + ) + self.is_base_dc = is_base_dc + + # Resolve paths for config directory and schema + curr_dir = os.path.dirname(os.path.abspath(__file__)) + target_config = config_dir or config_file_path or os.path.join(curr_dir, "configs") + schema_file_path = os.path.join(curr_dir, "schema.json") + + # Load and validate configuration + self.calculations = [] + if os.path.isdir(target_config): + yaml_files = sorted( + glob.glob(os.path.join(target_config, "*.yaml")) + glob.glob(os.path.join(target_config, "*.yml")) + ) + for file_path in yaml_files: + self.calculations.extend(validate_config(file_path, schema_file_path)) + else: + self.calculations = validate_config(target_config, schema_file_path) + + def run(self, active_imports: List[str], dry_run: bool = True) -> None: + """Executes aggregations independently for each active import. + + Blocks and synchronizes stage progression for each import: + Stage 1 -> Wait -> Stage 2 -> Wait -> Stage 3 -> Wait. + + Args: + active_imports: List of active import dataset names to process. + dry_run: If True, logs imports and active stages without executing BigQuery jobs. + """ + logging.info( + f"Starting Aggregation Orchestrator run (dry_run={dry_run}) for active imports: {active_imports}" + ) + + for single_import in active_imports: + logging.info(f"=== Starting Aggregation Pipeline for Import: '{single_import}' ===") + active_stages = self._get_active_stages_for_import(single_import) + + if not active_stages: + logging.info(f"No aggregation steps configured for import '{single_import}'. Skipping.") + continue + + if dry_run: + logging.info( + f"Detected active stage(s) {active_stages} for import '{single_import}'. Skipping execution because dry_run=True." + ) + continue + + for stage_num in active_stages: + logging.info(f"--- Triggering Stage {stage_num} for import '{single_import}' ---") + self._execute_and_synchronize_stage(single_import, stage_num) + + logging.info(f"=== Successfully completed all aggregation stages for Import: '{single_import}' ===") + + def _get_active_stages_for_import(self, single_import: str) -> List[int]: + """Returns a sorted list of unique active stage numbers for a single import. + + Args: + single_import: The active import dataset name. + + Returns: + Sorted list of stage numbers (e.g., [1, 2, 3]). + """ + stages = set() + for calc in self.calculations: + if self._calc_applies_to_import(calc, single_import): + stages.add(calc.get("stage", 1)) + return sorted(list(stages)) + + def get_active_stages(self, active_imports: List[str]) -> List[int]: + """Returns a sorted list of unique active stage numbers across active imports.""" + stages = set() + for single_import in active_imports: + stages.update(self._get_active_stages_for_import(single_import)) + return sorted(list(stages)) + + def execute_stage(self, stage_num: int, active_imports: List[str]) -> List[str]: + """Executes and collects BigQuery job IDs for a given stage. + + Args: + stage_num: The stage number to execute. + active_imports: List of active import dataset names. + + Returns: + List of BigQuery job IDs submitted. + """ + job_ids = [] + for single_import in active_imports: + jobs = self._dispatch_stage_steps(single_import, stage_num) + for job in jobs: + if job and getattr(job, "job_id", None): + job_ids.append(job.job_id) + return job_ids + + def _execute_and_synchronize_stage(self, single_import: str, stage_num: int) -> None: + """Triggers stage steps for a single import and blocks until completion.""" + jobs = self._dispatch_stage_steps(single_import, stage_num) + job_ids = [job.job_id for job in jobs if job and getattr(job, "job_id", None)] + + if not job_ids: + logging.info(f"No BigQuery jobs submitted for Stage {stage_num} (import: '{single_import}').") + return + + logging.info(f"Submitted {len(job_ids)} job(s) for Stage {stage_num} (import: '{single_import}'): {job_ids}") + self._wait_for_jobs(job_ids) + + def _dispatch_stage_steps(self, single_import: str, stage_num: int) -> List[Any]: + """Dispatches matching calculation steps for an import and stage number.""" + jobs = [] + for calc in self.calculations: + if calc.get("stage", 1) != stage_num: + continue + if not self._calc_applies_to_import(calc, single_import): + continue + + step_type = calc["type"] + calc_name = calc.get("name") or step_type + logging.info(f"Triggering step: '{calc_name}' (Stage {stage_num}) for import '{single_import}'...") + + step_jobs = [] + if step_type == "PLACE_AGGREGATION": + step_jobs = self._trigger_place(calc, [single_import]) + elif step_type == "STAT_VAR_AGGREGATION": + step_jobs = self._trigger_stat_var(calc, [single_import]) + elif step_type == "STAT_VAR_CALCULATION": + step_jobs = self._trigger_stat_var_calculation(calc, [single_import]) + elif step_type == "LINKED_EDGES": + step_jobs = self._trigger_linked_edges(calc, [single_import]) + elif step_type == "PROVENANCE_SUMMARY": + step_jobs = self._trigger_provenance_summary(calc, [single_import]) + elif step_type == "STAT_VAR_GROUPS": + step_jobs = self._trigger_stat_var_groups(calc, [single_import]) + else: + logging.warning( + f"Calculation type '{step_type}' configured for import '{single_import}' has no active generator handler." + ) + + jobs.extend(step_jobs) + return jobs + + def _wait_for_jobs(self, job_ids: List[str], poll_interval: int = 5) -> None: + """Blocks until all specified BigQuery job IDs complete successfully. + + Args: + job_ids: List of BigQuery job IDs to wait for. + poll_interval: Seconds between polling checks. + + Raises: + RuntimeError: If any job fails. + """ + if not job_ids: + return + + logging.info(f"Waiting for {len(job_ids)} BigQuery job(s) to complete: {job_ids}") + while True: + status_info = self.executor.get_jobs_status(job_ids) + status = status_info.get("status") + + if status == "DONE": + logging.info("All BigQuery jobs in stage completed successfully.") + return + elif status == "FAILED": + error_msg = status_info.get("error", "One or more BigQuery jobs failed.") + logging.error(f"Stage execution failed: {error_msg}") + raise RuntimeError(f"Aggregation execution failed: {error_msg}") + + time.sleep(poll_interval) + + def _trigger_place(self, config: Dict[str, Any], applicable_imports: List[str]) -> List[Any]: + """Triggers place-level rollup aggregations.""" + place_cfg = config.get("place_aggregation", {}) + from_type = place_cfg["from_place_types"] + to_type = place_cfg["to_place_types"] + + logging.info(f" -> Place Rollup: {from_type} -> {to_type} for imports {applicable_imports}") + generator = PlaceAggregationGenerator(self.executor, self.is_base_dc) + job = generator.aggregate_places( + import_names=applicable_imports, + source_type=from_type, + destination_type=to_type, + allow_multiple_to_places=place_cfg.get("allow_multiple_to_places", False) + ) + return [job] if job else [] + + def _trigger_stat_var(self, config: Dict[str, Any], applicable_imports: List[str]) -> List[Any]: + """Triggers statistical variable aggregations.""" + stat_cfg = config.get("stat_var_aggregation", {}) + aggregations = stat_cfg.get("aggregations", []) + output_import_name = config.get("output_import") + + generator = StatVarAggregator(self.executor, self.is_base_dc) + jobs = [] + + for item in aggregations: + ancestor_sv = item["ancestor_sv_id"] + source_svs = item["source_sv_ids"] + logging.info( + f" -> Stat Var Aggregation: ancestor '{ancestor_sv}' (sources: {source_svs}) for imports {applicable_imports}" + ) + item_jobs = generator.aggregate_stat_vars( + ancestor_sv=ancestor_sv, + source_svs=source_svs, + import_names=applicable_imports, + output_import_name=output_import_name, + skip_all_sources_present_check=item.get("skip_all_sources_present_check", False) + ) + jobs.extend(item_jobs) + + return jobs + + def _trigger_stat_var_calculation(self, config: Dict[str, Any], applicable_imports: List[str]) -> List[Any]: + """Triggers statistical variable calculations.""" + calc_cfg = config.get("stat_var_calculation", {}) + calculations = calc_cfg.get("calculations", []) + output_import_name = config.get("output_import") + + logging.info(f" -> Stat Var Calculation for imports {applicable_imports}") + generator = StatVarCalculationGenerator(self.executor, self.is_base_dc) + return generator.calculate_stat_vars( + calculations=calculations, + import_names=applicable_imports, + output_import_name=output_import_name + ) + + def _trigger_linked_edges(self, config: Dict[str, Any], applicable_imports: List[str]) -> List[Any]: + """Triggers linked edge aggregations.""" + logging.info(f" -> Linked Edges Aggregation for imports {applicable_imports}") + generator = LinkedEdgeGenerator(self.executor, self.is_base_dc) + return generator.run_all(applicable_imports) + + def _trigger_provenance_summary(self, config: Dict[str, Any], applicable_imports: List[str]) -> List[Any]: + """Triggers provenance summary aggregations.""" + logging.info(f" -> Provenance Summary Aggregation for imports {applicable_imports}") + generator = ProvenanceSummaryGenerator(self.executor, self.is_base_dc) + return generator.run_all(applicable_imports) + + def _trigger_stat_var_groups(self, config: Dict[str, Any], applicable_imports: List[str]) -> List[Any]: + """Triggers statistical variable group aggregations.""" + logging.info(f" -> Stat Var Groups Aggregation for imports {applicable_imports}") + generator = StatVarGroupGenerator(self.executor, self.is_base_dc) + return generator.run_all(applicable_imports) + + def _calc_applies_to_import(self, calc: Dict[str, Any], single_import: str) -> bool: + """Determines if a calculation step applies to a single import.""" + if calc.get("disabled", False): + return False + + configured_imports = calc.get("input_imports") or calc.get("imports", []) + if "*" in configured_imports or single_import in configured_imports: + return True + + return False diff --git a/pipeline/workflow/ingestion-helper/aggregation/orchestrator_test.py b/pipeline/workflow/ingestion-helper/aggregation/orchestrator_test.py new file mode 100644 index 000000000..95ed3e9e6 --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/orchestrator_test.py @@ -0,0 +1,173 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the AggregationOrchestrator class.""" + +import os +import sys +import tempfile +import textwrap +import unittest +from unittest.mock import MagicMock, patch + +sys.path.append(os.path.dirname(os.path.dirname(__file__))) + +from aggregation import AggregationOrchestrator + +VALID_CONFIG_YAML = textwrap.dedent("""\ + calculations: + - type: PLACE_AGGREGATION + input_imports: + - USFed_Census + output_import: USFed_Census_AggState + stage: 1 + place_aggregation: + from_place_types: County + to_place_types: State + + - type: STAT_VAR_AGGREGATION + input_imports: + - USFed_Census + output_import: USFed_Census_StatVarAgg + stage: 2 + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person + source_sv_ids: + - Count_Person_Male + - Count_Person_Female + skip_all_sources_present_check: true +""") + + +@patch('aggregation.orchestrator.BigQueryExecutor') +class TestOrchestratorScanning(unittest.TestCase): + """Tests stage scanning and active stage resolution methods.""" + + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + config_path = os.path.join(self.tmpdir.name, "config.yaml") + with open(config_path, "w") as f: + f.write(VALID_CONFIG_YAML) + + self.orchestrator = AggregationOrchestrator( + connection_id="conn", + project_id="proj", + instance_id="inst", + database_id="db", + config_file_path=config_path + ) + + def tearDown(self): + self.tmpdir.cleanup() + + def test_get_active_stages(self, mock_executor): + """Tests getting active stages for matching and non-matching imports.""" + stages = self.orchestrator.get_active_stages(["USFed_Census"]) + self.assertEqual(stages, [1, 2]) + + stages = self.orchestrator.get_active_stages(["OtherImport"]) + self.assertEqual(stages, []) + + def test_directory_config_loading(self, mock_executor): + """Tests that orchestrator correctly scans and loads config files from a directory.""" + dir_orchestrator = AggregationOrchestrator( + connection_id="conn", + project_id="proj", + instance_id="inst", + database_id="db", + config_dir=self.tmpdir.name + ) + self.assertEqual(len(dir_orchestrator.calculations), 2) + + +@patch('aggregation.orchestrator.BigQueryExecutor') +@patch('aggregation.orchestrator.PlaceAggregationGenerator') +@patch('aggregation.orchestrator.StatVarAggregator') +@patch('aggregation.orchestrator.StatVarCalculationGenerator') +class TestOrchestratorExecution(unittest.TestCase): + """Tests stage execution, verifying job submission and synchronization.""" + + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + config_path = os.path.join(self.tmpdir.name, "config.yaml") + with open(config_path, "w") as f: + f.write(VALID_CONFIG_YAML) + + self.orchestrator = AggregationOrchestrator( + connection_id="conn", + project_id="proj", + instance_id="inst", + database_id="db", + config_file_path=config_path + ) + + def tearDown(self): + self.tmpdir.cleanup() + + def test_run_dry_run_true(self, mock_calc_gen, mock_sv_agg, mock_place_gen, mock_executor_cls): + """Tests that run with dry_run=True logs imports and skips job submission.""" + self.orchestrator.run(active_imports=["USFed_Census"], dry_run=True) + mock_place_gen.return_value.aggregate_places.assert_not_called() + mock_sv_agg.return_value.aggregate_stat_vars.assert_not_called() + + def test_run_dry_run_false(self, mock_calc_gen, mock_sv_agg, mock_place_gen, mock_executor_cls): + """Tests that run with dry_run=False submits BigQuery jobs across stages.""" + mock_job1 = MagicMock() + mock_job1.job_id = "job-place-1" + mock_place_gen.return_value.aggregate_places.return_value = mock_job1 + + mock_job2 = MagicMock() + mock_job2.job_id = "job-sv-1" + mock_sv_agg.return_value.aggregate_stat_vars.return_value = [mock_job2] + + self.orchestrator.executor = MagicMock() + self.orchestrator.executor.get_jobs_status.return_value = {"status": "DONE"} + + self.orchestrator.run(active_imports=["USFed_Census"], dry_run=False) + + mock_place_gen.return_value.aggregate_places.assert_called_once_with( + import_names=["USFed_Census"], + source_type="County", + destination_type="State", + allow_multiple_to_places=False + ) + + mock_sv_agg.return_value.aggregate_stat_vars.assert_called_once_with( + ancestor_sv="Count_Person", + source_svs=["Count_Person_Male", "Count_Person_Female"], + import_names=["USFed_Census"], + output_import_name="USFed_Census_StatVarAgg", + skip_all_sources_present_check=True + ) + + def test_execute_stage(self, mock_calc_gen, mock_sv_agg, mock_place_gen, mock_executor_cls): + """Tests manual execution of a specific stage.""" + mock_job1 = MagicMock() + mock_job1.job_id = "job-place-1" + mock_place_gen.return_value.aggregate_places.return_value = mock_job1 + + job_ids = self.orchestrator.execute_stage(1, ["USFed_Census"]) + + mock_place_gen.return_value.aggregate_places.assert_called_once_with( + import_names=["USFed_Census"], + source_type="County", + destination_type="State", + allow_multiple_to_places=False + ) + self.assertEqual(job_ids, ["job-place-1"]) + + +if __name__ == '__main__': + unittest.main() diff --git a/pipeline/workflow/ingestion-helper/aggregation/schema.json b/pipeline/workflow/ingestion-helper/aggregation/schema.json new file mode 100644 index 000000000..a19047eb6 --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/schema.json @@ -0,0 +1,160 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AggregationCalculationsConfig", + "type": "object", + "required": ["calculations"], + "additionalProperties": false, + "properties": { + "calculations": { + "type": "array", + "items": { + "type": "object", + "required": ["type", "input_imports"], + "additionalProperties": false, + "properties": { + "name": { "type": "string" }, + "type": { + "type": "string", + "enum": [ + "PLACE_AGGREGATION", + "STAT_VAR_AGGREGATION", + "ENTITY_AGGREGATION", + "STAT_VAR_SERIES_AGGREGATION", + "STAT_VAR_CALCULATION", + "SUPER_ENUM_AGGREGATION", + "LINKED_EDGES", + "PROVENANCE_SUMMARY", + "STAT_VAR_GROUPS" + ] + }, + "input_imports": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "output_import": { "type": "string" }, + "stage": { "type": "integer" }, + "place_aggregation": { "type": "object" }, + "stat_var_aggregation": { "type": "object" }, + "entity_aggregation": { "type": "object" }, + "stat_var_series_aggregation": { "type": "object" }, + "stat_var_calculation": { "type": "object" }, + "disabled": { "type": "boolean" } + }, + "allOf": [ + { + "if": { "properties": { "type": { "const": "PLACE_AGGREGATION" } } }, + "then": { + "required": ["place_aggregation"], + "properties": { + "place_aggregation": { + "type": "object", + "required": ["from_place_types", "to_place_types"], + "properties": { + "from_place_types": { "type": "string" }, + "to_place_types": { "type": "string" }, + "allow_multiple_to_places": { "type": "boolean" } + } + } + } + } + }, + { + "if": { "properties": { "type": { "const": "STAT_VAR_AGGREGATION" } } }, + "then": { + "required": ["stat_var_aggregation"], + "properties": { + "stat_var_aggregation": { + "type": "object", + "required": ["aggregations"], + "properties": { + "aggregations": { + "type": "array", + "items": { + "type": "object", + "required": ["ancestor_sv_id", "source_sv_ids"], + "properties": { + "ancestor_sv_id": { "type": "string" }, + "source_sv_ids": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "skip_all_sources_present_check": { "type": "boolean" } + } + } + } + } + } + } + } + }, + { + "if": { "properties": { "type": { "const": "ENTITY_AGGREGATION" } } }, + "then": { + "required": ["entity_aggregation"], + "properties": { + "entity_aggregation": { + "type": "object", + "required": ["entity_types", "location_props", "date_prop", "agg_date_formats"], + "properties": { + "entity_types": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "location_props": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "date_prop": { "type": "string" }, + "agg_date_formats": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "constraints": { + "type": "array", + "items": { "type": "string" } + } + } + } + } + } + }, + { + "if": { "properties": { "type": { "const": "STAT_VAR_SERIES_AGGREGATION" } } }, + "then": { + "required": ["stat_var_series_aggregation"], + "properties": { + "stat_var_series_aggregation": { + "type": "object", + "required": ["aggr_funcs"], + "properties": { + "aggr_funcs": { "type": "array" } + } + } + } + } + }, + { + "if": { "properties": { "type": { "const": "STAT_VAR_CALCULATION" } } }, + "then": { + "required": ["stat_var_calculation"], + "properties": { + "stat_var_calculation": { + "type": "object", + "required": ["calculations"], + "properties": { + "calculations": { "type": "array" } + } + } + } + } + } + ] + } + } + } +} diff --git a/pipeline/workflow/ingestion-helper/aggregation/validator.py b/pipeline/workflow/ingestion-helper/aggregation/validator.py new file mode 100644 index 000000000..42d6dc2b1 --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/validator.py @@ -0,0 +1,153 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configuration validator and CLI tool for Data Commons aggregations.""" + +import argparse +import glob +import json +import logging +import os +import sys +from typing import Any, Dict, List +import yaml +import jsonschema + +# ANSI escape codes for colored terminal output +GREEN = "\033[92m" +RED = "\033[91m" +RESET = "\033[0m" + + +def validate_config(config_file_path: str, schema_file_path: str) -> List[Dict[str, Any]]: + """Loads and validates an aggregation YAML configuration file against the JSON Schema. + + Args: + config_file_path: Path to a .yaml configuration file. + schema_file_path: Path to the JSON Schema validation file. + + Returns: + A list of validated calculation dictionaries. + + Raises: + FileNotFoundError: If either the config or schema file is missing. + jsonschema.exceptions.ValidationError: If schema validation fails. + yaml.YAMLError: If the YAML file is malformed. + """ + if not os.path.exists(config_file_path): + raise FileNotFoundError(f"Aggregation config file not found: {config_file_path}") + if not os.path.exists(schema_file_path): + raise FileNotFoundError(f"JSON Schema file not found: {schema_file_path}") + + # 1. Load and parse YAML + try: + with open(config_file_path, "r") as f: + config = yaml.safe_load(f) + except yaml.YAMLError as e: + logging.error(f"Failed to parse YAML file {config_file_path}: {e}") + raise e + + if config is None: + config = {} + + # 2. Load JSON Schema + try: + with open(schema_file_path, "r") as f: + schema = json.load(f) + except Exception as e: + logging.error(f"Failed to load JSON Schema file {schema_file_path}: {e}") + raise e + + # 3. Validate against Schema + try: + jsonschema.validate(instance=config, schema=schema) + except jsonschema.exceptions.ValidationError as e: + logging.error(f"Schema validation failed for config {config_file_path}: {e.message}") + raise e + + return config.get("calculations", []) + + +def main(): + """CLI entry point for standalone configuration validation.""" + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser( + description="Validate Data Commons aggregation configuration files against the JSON Schema." + ) + + curr_dir = os.path.dirname(os.path.abspath(__file__)) + default_config = os.path.join(curr_dir, "configs") + default_schema = os.path.join(curr_dir, "schema.json") + + parser.add_argument( + "--config", + type=str, + default=default_config, + help=f"Path to an aggregation YAML config file or directory (default: {default_config})" + ) + parser.add_argument( + "--schema", + type=str, + default=default_schema, + help=f"Path to the JSON Schema validation file (default: {default_schema})" + ) + + args = parser.parse_args() + + # Collect target config files + if os.path.isdir(args.config): + yaml_files = sorted( + glob.glob(os.path.join(args.config, "*.yaml")) + glob.glob(os.path.join(args.config, "*.yml")) + ) + if not yaml_files: + print(f"{RED}[ERROR] No YAML config files found in directory: {args.config}{RESET}", file=sys.stderr) + sys.exit(1) + print(f"Validating {len(yaml_files)} configuration file(s) in '{args.config}' against '{os.path.basename(args.schema)}'...") + else: + yaml_files = [args.config] + print(f"Validating '{os.path.basename(args.config)}' against '{os.path.basename(args.schema)}'...") + + total_calculations = 0 + has_error = False + + for file_path in yaml_files: + try: + calculations = validate_config(file_path, args.schema) + total_calculations += len(calculations) + print(f" {GREEN}✓{RESET} {os.path.basename(file_path)} ({len(calculations)} calculation steps)") + except FileNotFoundError as e: + print(f" {RED}✗ {os.path.basename(file_path)} - File not found: {e}{RESET}", file=sys.stderr) + has_error = True + except jsonschema.exceptions.ValidationError as e: + print(f" {RED}✗ {os.path.basename(file_path)} - Schema Validation Failed:{RESET}", file=sys.stderr) + print(f" {RED}- Path: {'.'.join(str(p) for p in e.path)}{RESET}", file=sys.stderr) + print(f" {RED}- Message: {e.message}{RESET}", file=sys.stderr) + has_error = True + except yaml.YAMLError as e: + print(f" {RED}✗ {os.path.basename(file_path)} - YAML Syntax Error: {e}{RESET}", file=sys.stderr) + has_error = True + except Exception as e: + print(f" {RED}✗ {os.path.basename(file_path)} - Validation failure: {e}{RESET}", file=sys.stderr) + has_error = True + + if has_error: + print(f"\n{RED}[FAILURE] Validation failed for one or more configuration files.{RESET}", file=sys.stderr) + sys.exit(1) + else: + print(f"\n{GREEN}[SUCCESS] All {len(yaml_files)} configuration file(s) passed validation! ({total_calculations} calculation steps total){RESET}") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/pipeline/workflow/ingestion-helper/aggregation/validator_test.py b/pipeline/workflow/ingestion-helper/aggregation/validator_test.py new file mode 100644 index 000000000..01b01d4eb --- /dev/null +++ b/pipeline/workflow/ingestion-helper/aggregation/validator_test.py @@ -0,0 +1,325 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the aggregation configuration validator using real temporary files.""" + +import os +import sys +import tempfile +import textwrap +import unittest +import jsonschema +import yaml + +sys.path.append(os.path.dirname(os.path.dirname(__file__))) + +from aggregation import validate_config + + +class TestValidatorSuccess(unittest.TestCase): + """Verifies successful validation paths for valid configurations.""" + + def setUp(self): + self.schema_path = os.path.join(os.path.dirname(__file__), "schema.json") + self.tmpdir = tempfile.TemporaryDirectory() + self.config_path = os.path.join(self.tmpdir.name, "config.yaml") + + def tearDown(self): + self.tmpdir.cleanup() + + def test_validate_config_success_all_types(self): + """Verifies that a comprehensive, valid config with all 6 calculation types passes validation.""" + valid_all_types_yaml = textwrap.dedent("""\ + calculations: + - type: PLACE_AGGREGATION + input_imports: + - CensusACS5YearSurvey + output_import: CensusACS5YearSurvey_AggCountry + stage: 1 + place_aggregation: + from_place_types: State + to_place_types: Country + + - type: STAT_VAR_AGGREGATION + output_import: CensusACS5YearSurvey_HealthInsurance_StatVarAgg + input_imports: + - CensusACS5YearSurvey + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person_NoHealthInsurance + source_sv_ids: + - dc/y0dvhk0sggzef + + - type: ENTITY_AGGREGATION + output_import: FireFAMWEB_Agg + input_imports: + - FireFAMWEB + entity_aggregation: + entity_types: + - BurnedArea + location_props: + - location + date_prop: startDate + agg_date_formats: + - "%Y" + + - type: STAT_VAR_SERIES_AGGREGATION + input_imports: + - NASA_NEXDCP30 + stage: 1 + output_import: NASA_NEXDCP30_AggrDiffStats + stat_var_series_aggregation: + aggr_funcs: + - max_diff_across_measurement_methods: {} + + - type: STAT_VAR_CALCULATION + input_imports: + - EIA_Electricity + output_import: Energy_StatVarCalculation + stat_var_calculation: + calculations: + - input1: + sv_regex: Annual_Emissions + operation: DIVIDE + output: + sv: Annual_Emissions_Per_Capita + + - type: SUPER_ENUM_AGGREGATION + input_imports: + - CensusACS5YearSurvey + output_import: CensusACS5YearSurvey_SuperEnum + """) + + with open(self.config_path, "w") as f: + f.write(valid_all_types_yaml) + + calculations = validate_config(self.config_path, self.schema_path) + + self.assertEqual(len(calculations), 6) + self.assertEqual(calculations[0]["type"], "PLACE_AGGREGATION") + self.assertEqual(calculations[1]["type"], "STAT_VAR_AGGREGATION") + self.assertEqual(calculations[2]["type"], "ENTITY_AGGREGATION") + self.assertEqual(calculations[3]["type"], "STAT_VAR_SERIES_AGGREGATION") + self.assertEqual(calculations[4]["type"], "STAT_VAR_CALCULATION") + self.assertEqual(calculations[5]["type"], "SUPER_ENUM_AGGREGATION") + + +class TestValidatorSchemaConstraints(unittest.TestCase): + """Verifies core schema constraint failures (types, required fields, values).""" + + def setUp(self): + self.schema_path = os.path.join(os.path.dirname(__file__), "schema.json") + self.tmpdir = tempfile.TemporaryDirectory() + self.config_path = os.path.join(self.tmpdir.name, "config.yaml") + + def tearDown(self): + self.tmpdir.cleanup() + + def test_validate_config_missing_type(self): + """Verifies that missing the required 'type' field raises ValidationError.""" + invalid_missing_type_yaml = textwrap.dedent("""\ + calculations: + - input_imports: + - ImportA + """) + with open(self.config_path, "w") as f: + f.write(invalid_missing_type_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertEqual(ctx.exception.validator, "required") + + def test_validate_config_missing_input_imports(self): + """Verifies that missing the required 'input_imports' field raises ValidationError.""" + invalid_missing_imports_yaml = textwrap.dedent("""\ + calculations: + - type: SUPER_ENUM_AGGREGATION + """) + with open(self.config_path, "w") as f: + f.write(invalid_missing_imports_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("'input_imports' is a required property", ctx.exception.message) + + def test_validate_config_invalid_input_imports_type(self): + """Verifies that input_imports field being a string instead of an array raises ValidationError.""" + invalid_imports_type_yaml = textwrap.dedent("""\ + calculations: + - type: SUPER_ENUM_AGGREGATION + input_imports: "SingleImportString" + """) + with open(self.config_path, "w") as f: + f.write(invalid_imports_type_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("is not of type 'array'", ctx.exception.message) + + def test_validate_config_empty_input_imports_list(self): + """Verifies that an empty input_imports list raises ValidationError.""" + invalid_empty_imports_yaml = textwrap.dedent("""\ + calculations: + - type: SUPER_ENUM_AGGREGATION + input_imports: [] + """) + with open(self.config_path, "w") as f: + f.write(invalid_empty_imports_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("should be non-empty", ctx.exception.message) + + def test_validate_config_missing_calculations_key(self): + """Verifies that missing the required 'calculations' root key raises ValidationError.""" + missing_calculations_yaml = textwrap.dedent("""\ + some_other_key: [] + """) + with open(self.config_path, "w") as f: + f.write(missing_calculations_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("'calculations' is a required property", ctx.exception.message) + + def test_validate_config_empty_file(self): + """Verifies that a completely empty configuration file raises ValidationError.""" + empty_yaml = "" + with open(self.config_path, "w") as f: + f.write(empty_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("'calculations' is a required property", ctx.exception.message) + + +class TestValidatorConditionalDependencies(unittest.TestCase): + """Verifies type-specific conditional sub-block dependencies.""" + + def setUp(self): + self.schema_path = os.path.join(os.path.dirname(__file__), "schema.json") + self.tmpdir = tempfile.TemporaryDirectory() + self.config_path = os.path.join(self.tmpdir.name, "config.yaml") + + def tearDown(self): + self.tmpdir.cleanup() + + def test_validate_config_place_missing_subblock(self): + """Verifies that a PLACE_AGGREGATION step missing 'place_aggregation' raises ValidationError.""" + invalid_place_missing_yaml = textwrap.dedent("""\ + calculations: + - type: PLACE_AGGREGATION + input_imports: + - ImportA + """) + with open(self.config_path, "w") as f: + f.write(invalid_place_missing_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("'place_aggregation' is a required property", ctx.exception.message) + + def test_validate_config_stat_var_missing_subblock(self): + """Verifies that a STAT_VAR_AGGREGATION step missing 'stat_var_aggregation' raises ValidationError.""" + invalid_stat_var_missing_yaml = textwrap.dedent("""\ + calculations: + - type: STAT_VAR_AGGREGATION + input_imports: + - ImportA + """) + with open(self.config_path, "w") as f: + f.write(invalid_stat_var_missing_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("'stat_var_aggregation' is a required property", ctx.exception.message) + + def test_validate_config_stat_var_empty_source_svs(self): + """Verifies that an aggregation item with an empty source_sv_ids array raises ValidationError.""" + invalid_stat_var_empty_svs_yaml = textwrap.dedent("""\ + calculations: + - type: STAT_VAR_AGGREGATION + input_imports: + - ImportA + stat_var_aggregation: + aggregations: + - ancestor_sv_id: Count_Person + source_sv_ids: [] + """) + with open(self.config_path, "w") as f: + f.write(invalid_stat_var_empty_svs_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("should be non-empty", ctx.exception.message) + + def test_validate_config_entity_missing_subblock(self): + """Verifies that an ENTITY_AGGREGATION step missing 'entity_aggregation' raises ValidationError.""" + invalid_entity_missing_yaml = textwrap.dedent("""\ + calculations: + - type: ENTITY_AGGREGATION + input_imports: + - ImportA + """) + with open(self.config_path, "w") as f: + f.write(invalid_entity_missing_yaml) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: + validate_config(self.config_path, self.schema_path) + self.assertIn("'entity_aggregation' is a required property", ctx.exception.message) + + +class TestValidatorErrorsAndFileSystem(unittest.TestCase): + """Verifies file-system issues and non-schema parsing errors (YAML syntax).""" + + def setUp(self): + self.schema_path = os.path.join(os.path.dirname(__file__), "schema.json") + self.tmpdir = tempfile.TemporaryDirectory() + self.config_path = os.path.join(self.tmpdir.name, "config.yaml") + + def tearDown(self): + self.tmpdir.cleanup() + + def test_validate_config_yaml_syntax_error(self): + """Verifies that malformed YAML syntax raises YAMLError.""" + malformed_yaml = textwrap.dedent("""\ + calculations: + - type: PLACE_AGGREGATION + input_imports: + - "ImportA" + """) + with open(self.config_path, "w") as f: + f.write(malformed_yaml) + + with self.assertRaises(yaml.YAMLError): + validate_config(self.config_path, self.schema_path) + + def test_validate_config_missing_config_file(self): + """Verifies that a missing config file path raises FileNotFoundError.""" + with self.assertRaises(FileNotFoundError) as ctx: + validate_config("non_existent_config.yaml", self.schema_path) + self.assertIn("Aggregation config file not found", str(ctx.exception)) + + def test_validate_config_missing_schema_file(self): + """Verifies that a missing schema file path raises FileNotFoundError.""" + with open(self.config_path, "w") as f: + f.write("calculations: []") + + with self.assertRaises(FileNotFoundError) as ctx: + validate_config(self.config_path, "non_existent_schema.json") + self.assertIn("JSON Schema file not found", str(ctx.exception)) + + +if __name__ == '__main__': + unittest.main() diff --git a/pipeline/workflow/ingestion-helper/app_test.py b/pipeline/workflow/ingestion-helper/app_test.py index 8150e5a6b..acf0b7796 100644 --- a/pipeline/workflow/ingestion-helper/app_test.py +++ b/pipeline/workflow/ingestion-helper/app_test.py @@ -15,7 +15,6 @@ import unittest from unittest.mock import MagicMock, patch from datetime import datetime -import os from fastapi.testclient import TestClient from app import app @@ -263,6 +262,106 @@ def test_update_import_version_override_success(self, mock_get_caller_identity): # Verify get_caller_identity was called exactly once outside of the loop mock_get_caller_identity.assert_called_once() + @patch('routes.aggregation._get_orchestrator') + def test_aggregation_initiate_success(self, mock_aggregation_utils): + # Setup mock orchestrator + mock_instance = MagicMock() + mock_aggregation_utils.return_value = mock_instance + mock_instance.get_active_stages.return_value = [1] + mock_instance.execute_stage.return_value = ["job-1", "job-2"] + + # Call endpoint + payload = { + "importList": [{"importName": "USFed_Census"}] + } + response = client.post("/aggregation/initiate", json=payload) + + # Assertions + self.assertEqual(response.status_code, 200) + state = response.json() + self.assertEqual(state["status"], "RUNNING") + self.assertEqual(state["current_stage"], 1) + self.assertEqual(state["active_job_ids"], ["job-1", "job-2"]) + self.assertEqual(state["import_list"], [{"importName": "USFed_Census"}]) + + @patch('routes.aggregation._get_orchestrator') + def test_aggregation_poll_transition(self, mock_aggregation_utils): + # Setup mock orchestrator to simulate Stage 1 completion and Stage 2 execution + mock_instance = MagicMock() + mock_aggregation_utils.return_value = mock_instance + mock_instance.get_active_stages.return_value = [1, 2] + + # Mock BQ reporting Stage 1 jobs are DONE + mock_instance.check_jobs_status.return_value = {"status": "DONE"} + mock_instance.execute_stage.return_value = ["job-stage2-1"] + + # Input state (Stage 1 completed) + payload = { + "status": "RUNNING", + "current_stage": 1, + "active_job_ids": ["job-1", "job-2"], + "import_list": [{"importName": "USFed_Census"}] + } + + # Call endpoint + response = client.post("/aggregation/poll", json=payload) + + # Assertions + self.assertEqual(response.status_code, 200) + state = response.json() + self.assertEqual(state["status"], "RUNNING") + self.assertEqual(state["current_stage"], 2) # Transitioned to 2! + self.assertEqual(state["active_job_ids"], ["job-stage2-1"]) + + @patch('routes.aggregation._get_orchestrator') + def test_aggregation_poll_still_running(self, mock_aggregation_utils): + # Setup mock orchestrator to simulate jobs still in PENDING state + mock_instance = MagicMock() + mock_aggregation_utils.return_value = mock_instance + mock_instance.get_active_stages.return_value = [1] + + # Mock BQ reporting Stage 1 jobs are PENDING (still executing) + mock_instance.check_jobs_status.return_value = {"status": "PENDING"} + + # Input state + payload = { + "status": "RUNNING", + "current_stage": 1, + "active_job_ids": ["job-1", "job-2"], + "import_list": [{"importName": "USFed_Census"}] + } + + # Call endpoint + response = client.post("/aggregation/poll", json=payload) + + # Assertions + self.assertEqual(response.status_code, 200) + state = response.json() + # Verify state is returned unchanged + self.assertEqual(state["status"], "RUNNING") + self.assertEqual(state["current_stage"], 1) + self.assertEqual(state["active_job_ids"], ["job-1", "job-2"]) + + @patch('routes.aggregation._get_orchestrator') + def test_aggregation_run(self, mock_aggregation_utils): + # Setup mock orchestrator + mock_instance = MagicMock() + mock_aggregation_utils.return_value = mock_instance + mock_instance.get_active_stages.return_value = [1, 2] + mock_instance.execute_stage.side_effect = lambda stage, imports: [f"job-stage{stage}-1"] + + # Call legacy endpoint + payload = { + "importList": [{"importName": "USFed_Census"}] + } + response = client.post("/aggregation/run", json=payload) + + # Assertions (should return all jobs from all stages in parallel) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data["status"], "SUBMITTED") + self.assertEqual(data["jobIds"], ["job-stage1-1", "job-stage2-1"]) + if __name__ == '__main__': unittest.main() diff --git a/pipeline/workflow/ingestion-helper/pyproject.toml b/pipeline/workflow/ingestion-helper/pyproject.toml index 64cd6f4b0..832db88c1 100644 --- a/pipeline/workflow/ingestion-helper/pyproject.toml +++ b/pipeline/workflow/ingestion-helper/pyproject.toml @@ -33,6 +33,8 @@ dependencies = [ "google-cloud-bigquery", "redis", "jinja2", + "pyyaml>=6.0.3", + "jsonschema>=4.26.0", "pandas~=3.0.3", ] diff --git a/pipeline/workflow/ingestion-helper/utils/aggregation.py b/pipeline/workflow/ingestion-helper/utils/aggregation.py deleted file mode 100644 index 432a4000f..000000000 --- a/pipeline/workflow/ingestion-helper/utils/aggregation.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import Any, Dict, List, Optional - -from aggregation import BigQueryExecutor -from aggregation import LinkedEdgeGenerator -from aggregation import ProvenanceSummaryGenerator -from aggregation import StatVarAggregator -from aggregation import StatVarGroupGenerator -from google.cloud import bigquery - -logging.getLogger().setLevel(logging.INFO) - - -class AggregationUtils: - """Orchestrates the overall aggregation workflow.""" - - def __init__(self, - connection_id: str, - project_id: str, - instance_id: str, - database_id: str, - location: Optional[str] = None, - is_base_dc: bool = True) -> None: - # TODO: remove sequential execution once DCP changes are made - # Use sequential execution for DCP (backward compatibility) - run_sequential = not is_base_dc - self.executor = BigQueryExecutor(connection_id=connection_id, - project_id=project_id, - instance_id=instance_id, - database_id=database_id, - location=location, - run_sequential=run_sequential) - self.linked_edge_generator = LinkedEdgeGenerator( - self.executor, is_base_dc) - self.provenance_summary_generator = ProvenanceSummaryGenerator( - self.executor, is_base_dc) - - def run_aggregation(self, import_list: List[Dict[str, Any]]) -> List[str]: - """ - Orchestrates standard per-import aggregations and global aggregations. - Returns a list of BigQuery job IDs for async polling. - """ - logging.info(f"Received request for importList: {import_list}") - - try: - import_names = [] - # 1. Run standard per-import aggregations - for import_item in import_list: - import_name = import_item.get('importName') - if import_name: - import_names.append(import_name) - query = "SELECT @import_name as import_name, CURRENT_TIMESTAMP() as execution_time" - job_config = bigquery.QueryJobConfig(query_parameters=[ - bigquery.ScalarQueryParameter("import_name", "STRING", - import_name), - ]) - self.executor.execute(query, job_config=job_config) - else: - logging.info( - 'Skipping aggregation logic for empty importName') - - # 2. Run global aggregations asynchronously - jobs = [] - jobs.extend(self.linked_edge_generator.run_all(import_names)) - jobs.extend(self.provenance_summary_generator.run_all(import_names)) - - job_ids = [job.job_id for job in jobs if job] - logging.info(f"Submitted async aggregation jobs: {job_ids}") - - return job_ids - except Exception as e: - logging.error(f"Aggregation failed: {e}") - raise e - - def check_aggregation_status(self, job_ids: List[str]) -> Dict[str, Any]: - """ - Checks the status of the provided BigQuery job IDs. - """ - logging.info(f"Checking status for jobs: {job_ids}") - try: - return self.executor.get_jobs_status(job_ids) - except Exception as e: - logging.error(f"Failed to check aggregation status: {e}") - raise e diff --git a/pipeline/workflow/ingestion-helper/utils/aggregation_test.py b/pipeline/workflow/ingestion-helper/utils/aggregation_test.py deleted file mode 100644 index f4afb6e3f..000000000 --- a/pipeline/workflow/ingestion-helper/utils/aggregation_test.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import unittest -from unittest.mock import MagicMock -from unittest.mock import patch - -sys.path.append(os.path.dirname(os.path.dirname(__file__))) - -from utils.aggregation import AggregationUtils - - -@patch('utils.aggregation.BigQueryExecutor') -@patch('utils.aggregation.LinkedEdgeGenerator') -@patch('utils.aggregation.ProvenanceSummaryGenerator') -@patch('utils.aggregation.StatVarGroupGenerator') -class TestAggregationUtils(unittest.TestCase): - - def test_run_aggregation(self, mock_prov_gen, mock_edge_gen, mock_executor): - # Setup mocks - mock_executor_instance = MagicMock() - mock_executor.return_value = mock_executor_instance - - mock_edge_gen_instance = MagicMock() - mock_edge_gen.return_value = mock_edge_gen_instance - mock_job1 = MagicMock() - mock_job1.job_id = "job1" - mock_edge_gen_instance.run_all.return_value = [mock_job1] - - mock_prov_gen_instance = MagicMock() - mock_prov_gen.return_value = mock_prov_gen_instance - mock_job2 = MagicMock() - mock_job2.job_id = "job2" - mock_prov_gen_instance.run_all.return_value = [mock_job2] - - utils = AggregationUtils(connection_id="conn", - project_id="proj", - instance_id="inst", - database_id="db", - is_base_dc=True) - - import_list = [{'importName': 'import1'}, {'importName': 'import2'}] - job_ids = utils.run_aggregation(import_list) - - # Verify standard import queries were executed - self.assertEqual(mock_executor_instance.execute.call_count, 2) - - # Verify generators were called - mock_edge_gen_instance.run_all.assert_called_once_with( - ["import1", "import2"]) - mock_prov_gen_instance.run_all.assert_called_once_with( - ["import1", "import2"]) - - self.assertEqual(job_ids, ["job1", "job2"]) - - def test_check_aggregation_status(self, mock_prov_gen, mock_edge_gen, - mock_executor): - mock_executor_instance = MagicMock() - mock_executor.return_value = mock_executor_instance - mock_executor_instance.get_jobs_status.return_value = {"status": "DONE"} - - utils = AggregationUtils(connection_id="conn", - project_id="proj", - instance_id="inst", - database_id="db") - - status = utils.check_aggregation_status(["job1", "job2"]) - mock_executor_instance.get_jobs_status.assert_called_once_with( - ["job1", "job2"]) - self.assertEqual(status, {"status": "DONE"}) - - -if __name__ == '__main__': - unittest.main() diff --git a/pipeline/workflow/ingestion-helper/uv.lock b/pipeline/workflow/ingestion-helper/uv.lock index 3070a005a..2a3532cb0 100644 --- a/pipeline/workflow/ingestion-helper/uv.lock +++ b/pipeline/workflow/ingestion-helper/uv.lock @@ -53,6 +53,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "certifi" version = "2026.5.20" @@ -278,8 +287,10 @@ dependencies = [ { name = "google-cloud-spanner" }, { name = "google-cloud-storage" }, { name = "jinja2" }, + { name = "jsonschema" }, { name = "pandas" }, { name = "pydantic" }, + { name = "pyyaml" }, { name = "redis" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -300,8 +311,10 @@ requires-dist = [ { name = "google-cloud-spanner" }, { name = "google-cloud-storage" }, { name = "jinja2" }, + { name = "jsonschema", specifier = ">=4.26.0" }, { name = "pandas", specifier = "~=3.0.3" }, { name = "pydantic", specifier = ">=2.0.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "redis" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.28.0" }, ] @@ -730,6 +743,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -1307,6 +1347,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/e3/b519734372d305bd547534a9f32e4ce9f98552af753dce72cf3483a0ff0b/redis-8.0.0-py3-none-any.whl", hash = "sha256:c938c18338585009f0bc310f4c7e4e4b4d37639356c4ac072cedf3af570c8dc7", size = 499870, upload-time = "2026-05-28T12:45:11.697Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1322,6 +1376,102 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, +] + [[package]] name = "six" version = "1.17.0"