Add State and household size as rent predictors (#82)

* Add household size and state to predictors

* Adjust reweighting algorithm

* Adjust learning rate

* Remove <10k AGI taxable targets and update documentation

* Update dataset locations

Author: nikhilwoodruff

changelog_entry.yaml

@@ -0,0 +1,4 @@
+- bump: minor
+  changes:
+    added:
+    - State and household size as predictors for rent and property taxes.

docs/validation.ipynb

@@ -81,6 +81,8 @@ def add_rent(self, cps: h5py.File, person: DataFrame, household: DataFrame):
         "self_employment_income",
         "social_security",
         "pension_income",
+        "state_code_str",
+        "household_size",
     ]
     IMPUTATIONS = ["rent", "real_estate_taxes"]
     train_df = acs.calculate_dataframe(PREDICTORS + IMPUTATIONS)
@@ -593,7 +595,7 @@ class CPS_2021(CPS):
     label = "CPS 2021"
     raw_cps = CensusCPS_2021
     previous_year_raw_cps = CensusCPS_2020
-    file_path = STORAGE_FOLDER / "cps_2021.h5"
+    file_path = STORAGE_FOLDER / "cps_2021_v1_6_1.h5"
     time_period = 2021
 
 
@@ -602,7 +604,7 @@ class CPS_2022(CPS):
     label = "CPS 2022"
     raw_cps = CensusCPS_2022
     previous_year_raw_cps = CensusCPS_2021
-    file_path = STORAGE_FOLDER / "cps_2022.h5"
+    file_path = STORAGE_FOLDER / "cps_2022_v1_6_1.h5"
     time_period = 2022
 
 
@@ -618,9 +620,9 @@ class CPS_2023(CPS):
 class CPS_2024(CPS):
     name = "cps_2024"
     label = "CPS 2024 (2022-based)"
-    file_path = STORAGE_FOLDER / "cps_2024.h5"
+    file_path = STORAGE_FOLDER / "cps_2024_v1_6_1.h5"
     time_period = 2024
-    url = "release://policyengine/policyengine-us-data/release/cps_2024.h5"
+    url = "release://policyengine/policyengine-us-data/release/cps_2024_v1_6_1.h5"
 
 
 class PooledCPS(Dataset):
@@ -672,14 +674,14 @@ def generate(self):
 class Pooled_3_Year_CPS_2023(PooledCPS):
     label = "CPS 2023 (3-year pooled)"
     name = "pooled_3_year_cps_2023"
-    file_path = STORAGE_FOLDER / "pooled_3_year_cps_2023.h5"
+    file_path = STORAGE_FOLDER / "pooled_3_year_cps_2023_v1_6_1.h5"
     input_datasets = [
         CPS_2021,
         CPS_2022,
         CPS_2023,
     ]
     time_period = 2023
-    url = "release://PolicyEngine/policyengine-us-data/release/pooled_3_year_cps_2023.h5"
+    url = "release://PolicyEngine/policyengine-us-data/release/pooled_3_year_cps_2023_v1_6_1.h5"
 
 
 if __name__ == "__main__":

policyengine_us_data/datasets/cps/cps.py

@@ -48,20 +48,23 @@ def loss(weights):
         ) ** 2
         if torch.isnan(rel_error).any():
             raise ValueError("Relative error contains NaNs")
-        worst_name = target_names[torch.argmax(rel_error)]
-        worst_val = rel_error[torch.argmax(rel_error)].item()
-        return rel_error.mean(), worst_name, worst_val
+        return rel_error.mean()
 
-    optimizer = torch.optim.Adam([weights], lr=1e-2)
+    optimizer = torch.optim.Adam([weights], lr=1e-1)
     from tqdm import trange
 
-    iterator = trange(10_000)
+    start_loss = None
+
+    iterator = trange(1_000)
     for i in iterator:
         optimizer.zero_grad()
-        l, worst_name, worst_val = loss(torch.exp(weights))
+        l = loss(torch.exp(weights))
+        if start_loss is None:
+            start_loss = l.item()
+        loss_rel_change = (l.item() - start_loss) / start_loss
         l.backward()
         iterator.set_postfix(
-            {"loss": l.item(), "worst": worst_name, "val": worst_val}
+            {"loss": l.item(), "loss_rel_change": loss_rel_change}
         )
         optimizer.step()
 
@@ -173,8 +176,8 @@ class EnhancedCPS_2024(EnhancedCPS):
     end_year = 2024
     name = "enhanced_cps_2024"
     label = "Enhanced CPS 2024"
-    file_path = STORAGE_FOLDER / "enhanced_cps_2024.h5"
-    url = "release://policyengine/policyengine-us-data/release/enhanced_cps_2024.h5"
+    file_path = STORAGE_FOLDER / "enhanced_cps_2024_v1_6_1.h5"
+    url = "release://policyengine/policyengine-us-data/release/enhanced_cps_2024_v1_6_1.h5"
 
 
 if __name__ == "__main__":

policyengine_us_data/datasets/cps/enhanced_cps.py

@@ -71,6 +71,9 @@ def build_loss_matrix(dataset: type, time_period):
         if not row["Taxable only"]:
             continue  # exclude non "taxable returns" statistics
 
+        if row["AGI upper bound"] <= 10_000:
+            continue
+
         mask = (
             (agi >= row["AGI lower bound"])
             * (agi < row["AGI upper bound"])
@@ -180,6 +183,17 @@ def build_loss_matrix(dataset: type, time_period):
             ).calibration.gov.cbo._children[variable_name]
         )
 
+    # Treasury EITC
+
+    loss_matrix["treasury/eitc"] = sim.calculate(
+        "eitc", map_to="household"
+    ).values
+    targets_array.append(
+        sim.tax_benefit_system.parameters(
+            time_period
+        ).calibration.gov.treasury.tax_expenditures.eitc
+    )
+
     # CPS-derived statistics
     # Medical expenses, sum of spm thresholds
     # Child support expenses