diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d4f94d52..fbeddcfc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,9 @@ +## Summary +What does this PR do? + +## Main file changes +Summarise changes to main files to be reviewed. + ## Checklist Before you mark your PR as ready for review, please ensure you have completed the following. diff --git a/causal_testing/__main__.py b/causal_testing/__main__.py index ddc9641b..5053acb8 100644 --- a/causal_testing/__main__.py +++ b/causal_testing/__main__.py @@ -79,7 +79,15 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: "-s", "--silent", action="store_true", - help="Do not crash on error. If set to true, errors are recorded as test results.", + help="Do not crash on error. If set to true, errors are recorded as test results. (Defaults to False)", + default=False, + ) + parser_test.add_argument( + "-R", + "--include-adequacy-results", + action="store_true", + help="Include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy " + "bootstraps. (Defaults to False)", default=False, ) @@ -257,7 +265,7 @@ def main() -> None: logging.info("Running tests") framework.run_tests(silent=args.silent, adequacy=args.adequacy, bootstrap_size=args.bootstrap_size) - framework.save_results(args.output) + framework.save_results(args.output, include_adequacy_results=args.include_adequacy_results) logging.info("Causal testing completed successfully.") case Command.EVALUATE: diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py index 016b1d61..ddb59563 100644 --- a/causal_testing/causal_testing_framework.py +++ b/causal_testing/causal_testing_framework.py @@ -145,45 +145,33 @@ def create_causal_test(self, test: dict) -> CausalTestCase: :return: CausalTestCase object :raises: ValueError if invalid estimator or configuration is provided """ + # Create the estimator with correct parameters estimator_map = {ff.name: ff for ff in entry_points(group="estimators")} - effect_map = {ff.name: ff for ff in entry_points(group="causal_effects")} - if "estimator" not in test: - raise ValueError("Test configuration must specify an estimator") - - if test["estimator"] not in estimator_map: + raise ValueError("Test configuration must specify an estimator.") + estimator_kwargs = test["estimator"] + estimator_name = estimator_kwargs.pop("name") + if estimator_name not in estimator_map: raise ValueError( - f"Unsupported estimator {test['estimator']}. Supported: {sorted(estimator_map)}. " + f"Unsupported estimator {estimator_name}. Supported: {sorted(estimator_map)}. " "If you have implemented a custom estimator, you will need to add this to your entrypoints via your " "pyproject.toml file." ) + estimator = estimator_map.get(estimator_name).load()(**estimator_kwargs) - # Create the estimator with correct parameters - treatment_variable = test.get("treatment_variable") - outcome_variable = test.get("outcome_variable") - estimator_class = estimator_map.get(test["estimator"]).load() - estimator_kwargs = test.get("estimator_kwargs", {}) - effect_type = test.get("expected_effect", {}).get("effect_type", "direct") - - estimator = estimator_class( - treatment_variable=treatment_variable, - outcome_variable=outcome_variable, - treatment_value=test.get("treatment_value"), - control_value=test.get("control_value"), - alpha=test.get("alpha", 0.05), - **estimator_kwargs, - ) - - # Get effect type and create expected effect - expected_effect = test["expected_effect"] - effect_type = expected_effect.pop("name") - if effect_type not in effect_map: + # Create an effect with the corect parameters + effect_map = {ff.name: ff for ff in entry_points(group="causal_effects")} + if "expected_effect" not in test: + raise ValueError("Test configuration must specify an expected effect.") + expected_effect_kwargs = test["expected_effect"] + expected_effect_name = expected_effect_kwargs.pop("name") + if expected_effect_name not in effect_map: raise ValueError( - f"Unsupported causal effect {effect_type}. Supported: {sorted(effect_map)}. " + f"Unsupported causal effect {expected_effect_name}. Supported: {sorted(effect_map)}. " "If you have implemented a custom causal effect, you will need to add this to your entrypoints via " "your pyproject.toml file." ) - expected_effect = effect_map[effect_type].load()(**expected_effect) + expected_effect = effect_map[expected_effect_name].load()(**expected_effect_kwargs) return CausalTestCase( name=test.get("name"), @@ -267,8 +255,14 @@ def evaluate_dag(self, bootstrap_size: bool = 100, alpha: float = 0.05) -> pd.Se return pd.Series(results).sort_index() - def save_results(self, output_path) -> list: - """Save test results to JSON file in the expected format.""" + def save_results(self, output_path: str, include_adequacy_results: bool = False): + """ + Save test results to JSON file in the expected format. + + :param output_path: Path for output file (.json). + :param include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy + bootstraps. + """ logger.info(f"Saving results to {output_path}") # Create parent directory if it doesn't exist @@ -276,6 +270,10 @@ def save_results(self, output_path) -> list: # Save to file with open(output_path, "w", encoding="utf-8") as f: - json.dump([test.to_dict() for test in self.test_cases], f, indent=2) + json.dump( + [test.to_dict(include_adequacy_results=include_adequacy_results) for test in self.test_cases], + f, + indent=2, + ) logger.info("Results saved successfully") diff --git a/causal_testing/estimation/instrumental_variable_estimator.py b/causal_testing/estimation/instrumental_variable_estimator.py index f4ffab37..e52515dc 100644 --- a/causal_testing/estimation/instrumental_variable_estimator.py +++ b/causal_testing/estimation/instrumental_variable_estimator.py @@ -25,8 +25,6 @@ def __init__( self, outcome_variable: str, treatment_variable: str, - treatment_value: float, - control_value: float, instrument: str, alpha: float = 0.05, bootstrap_size=100, @@ -34,8 +32,6 @@ def __init__( super().__init__( treatment_variable=treatment_variable, outcome_variable=outcome_variable, - treatment_value=treatment_value, - control_value=control_value, alpha=alpha, ) @@ -47,13 +43,17 @@ def add_modelling_assumptions(self): Add modelling assumptions to the estimator. This is a list of strings which list the modelling assumptions that must hold if the resulting causal inference is to be considered valid. """ - self.modelling_assumptions.append("""The instrument and the treatment, and the treatment and the outcome must be - related linearly in the form Y = aX + b.""") - self.modelling_assumptions.append("""The three IV conditions must hold + self.modelling_assumptions.append( + """The instrument and the treatment, and the treatment and the outcome must be + related linearly in the form Y = aX + b.""" + ) + self.modelling_assumptions.append( + """The three IV conditions must hold (i) Instrument is associated with treatment (ii) Instrument does not affect outcome except through its potential effect on treatment (iii) Instrument and outcome do not share causes - """) + """ + ) def iv_coefficient(self, df) -> float: """ diff --git a/causal_testing/estimation/linear_regression_estimator.py b/causal_testing/estimation/linear_regression_estimator.py index deba16ce..2c91ded7 100644 --- a/causal_testing/estimation/linear_regression_estimator.py +++ b/causal_testing/estimation/linear_regression_estimator.py @@ -155,7 +155,7 @@ def estimate_ate_calculated(self, df: pd.DataFrame) -> EffectEstimate: return EffectEstimate("ate", pd.Series(treatment_outcome["mean"] - control_outcome["mean"]), ci_low, ci_high) def _get_confidence_intervals(self, model, treatment): - confidence_intervals = model.conf_int(alpha=self.alpha, cols=None) + confidence_intervals = model.conf_int(alpha=self.alpha) ci_low, ci_high = ( pd.Series(confidence_intervals[0].loc[treatment]), pd.Series(confidence_intervals[1].loc[treatment]), diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index ec1de7b8..4531b67d 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -172,10 +172,11 @@ def estimate_effect(self, df: pd.DataFrame) -> CausalTestResult: estimate_effect = getattr(self.estimator, f"estimate_{self.effect_measure}") return estimate_effect(df) - def to_dict(self) -> dict: + def to_dict(self, include_adequacy_results: bool = False) -> dict: """ Convert the test case to a python dictionary for easy serialisation as JSON. + :bool include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy bootstraps. :returns: A JSON serialisable dict representing the test case. """ test_case = { @@ -185,12 +186,11 @@ def to_dict(self) -> dict: "query": self.query, } - for label, attribute in [ - ("expected_effect", self.expected_causal_effect), - ("estimator", self.estimator), - ("result", self.result), - ]: - if attribute is not None: - test_case[label] = attribute.to_dict() + if self.expected_causal_effect is not None: + test_case["expected_effect"] = self.expected_causal_effect.to_dict() + if self.estimator is not None: + test_case["estimator"] = self.estimator.to_dict() + if self.result is not None: + test_case["result"] = self.result.to_dict(include_adequacy_results=include_adequacy_results) return test_case diff --git a/causal_testing/testing/causal_test_result.py b/causal_testing/testing/causal_test_result.py index 5bfb15c1..c5240943 100644 --- a/causal_testing/testing/causal_test_result.py +++ b/causal_testing/testing/causal_test_result.py @@ -34,10 +34,12 @@ def passed(self) -> bool: """ return self.outcome == TestOutcome.PASS - def to_dict(self): + def to_dict(self, include_adequacy_results: bool = False): """ Convert the result to a python dictionary for easy serialisation as JSON. + :param include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy + bootstraps. :returns: A JSON serialisable dict representing the test result. """ @@ -47,6 +49,6 @@ def to_dict(self): effect_estimate = self.effect_estimate.to_dict() if self.effect_estimate else {} - adequacy = self.adequacy.to_dict() if self.adequacy else {} + adequacy = self.adequacy.to_dict(include_adequacy_results=include_adequacy_results) if self.adequacy else {} return outcome | effect_estimate | {"adequacy": adequacy} diff --git a/causal_testing/testing/data_adequacy.py b/causal_testing/testing/data_adequacy.py index 192ef1a1..b094e554 100644 --- a/causal_testing/testing/data_adequacy.py +++ b/causal_testing/testing/data_adequacy.py @@ -34,10 +34,10 @@ def __init__( self.successful = successful self.bootstrap_size = bootstrap_size - def to_dict(self, include_results: bool = False): + def to_dict(self, include_adequacy_results: bool = False): """ :returns: the adequacy object as a dictionary. - :param include_results: Whether to serialise the results. + :param include_adequacy_results: Whether to serialise the results. """ result = { "kurtosis": self.kurtosis.to_dict(), @@ -45,6 +45,6 @@ def to_dict(self, include_results: bool = False): "successful": self.successful, "bootstrap_size": self.bootstrap_size, } - if include_results: + if include_adequacy_results: return result | {"results": self.results.reset_index(drop=True).to_dict()} return result diff --git a/docs/source/tutorials/vaccinating_elderly/causal_tests.json b/docs/source/tutorials/vaccinating_elderly/causal_tests.json index 3c4656c0..4c8409a4 100644 --- a/docs/source/tutorials/vaccinating_elderly/causal_tests.json +++ b/docs/source/tutorials/vaccinating_elderly/causal_tests.json @@ -1,109 +1,309 @@ { - "tests": [ - { - "name": "max_doses _||_ cum_vaccinations", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "tests": [{ + "name": "max_doses _||_ vaccine", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "max_doses", + "outcome_variable": "vaccine", + "alpha": 0.05, + "formula": "vaccine ~ max_doses" + } + }, { + "name": "vaccine _||_ max_doses", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "vaccine", + "outcome_variable": "max_doses", + "alpha": 0.05, + "formula": "max_doses ~ vaccine" + } + }, { + "name": "max_doses _||_ cum_vaccinations", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "max_doses", "outcome_variable": "cum_vaccinations", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinations ~ max_doses"}, "alpha": 0.05, - "skip": false + "formula": "cum_vaccinations ~ max_doses" + } + }, { + "name": "cum_vaccinations _||_ max_doses", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_vaccinations", + "outcome_variable": "max_doses", + "alpha": 0.05, + "formula": "max_doses ~ cum_vaccinations" + } + }, { + "name": "max_doses _||_ cum_vaccinated", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 }, - { - "name": "max_doses _||_ cum_vaccinated", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "max_doses", "outcome_variable": "cum_vaccinated", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinated ~ max_doses"}, "alpha": 0.05, - "skip": false + "formula": "cum_vaccinated ~ max_doses" + } + }, { + "name": "cum_vaccinated _||_ max_doses", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_vaccinated", + "outcome_variable": "max_doses", + "alpha": 0.05, + "formula": "max_doses ~ cum_vaccinated" + } + }, { + "name": "max_doses _||_ cum_infections", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 }, - { - "name": "max_doses _||_ cum_infections", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "max_doses", "outcome_variable": "cum_infections", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_infections ~ max_doses"}, "alpha": 0.05, - "skip": false + "formula": "cum_infections ~ max_doses" + } + }, { + "name": "cum_infections _||_ max_doses", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_infections", + "outcome_variable": "max_doses", + "alpha": 0.05, + "formula": "max_doses ~ cum_infections" + } + }, { + "name": "vaccine -> cum_vaccinations", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "SomeEffect", + "effect_type": "direct" }, - { - "name": "vaccine --> cum_vaccinations", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "vaccine", "outcome_variable": "cum_vaccinations", - "expected_effect": {"name": "SomeEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinations ~ vaccine"}, - "skip": false - }, - { - "name": "vaccine --> cum_vaccinated", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "alpha": 0.05, + "formula": "cum_vaccinations ~ vaccine" + } + }, { + "name": "vaccine -> cum_vaccinated", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "SomeEffect", + "effect_type": "direct" + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "vaccine", "outcome_variable": "cum_vaccinated", - "expected_effect": {"name": "SomeEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinated ~ vaccine"}, - "skip": false - }, - { - "name": "vaccine --> cum_infections", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "alpha": 0.05, + "formula": "cum_vaccinated ~ vaccine" + } + }, { + "name": "vaccine -> cum_infections", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "SomeEffect", + "effect_type": "direct" + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "vaccine", "outcome_variable": "cum_infections", - "expected_effect": {"name": "SomeEffect"}, - "estimator_kwargs": {"formula": "cum_infections ~ vaccine"}, - "skip": false - }, - { - "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "alpha": 0.05, + "formula": "cum_infections ~ vaccine" + } + }, { + "name": "cum_vaccinations _||_ cum_vaccinated", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "cum_vaccinations", "outcome_variable": "cum_vaccinated", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_vaccinated ~ cum_vaccinations + vaccine"}, "alpha": 0.05, - "skip": false + "adjustment_set": ["vaccine"], + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine" + } + }, { + "name": "cum_vaccinated _||_ cum_vaccinations", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 }, - { - "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_vaccinated", + "outcome_variable": "cum_vaccinations", + "alpha": 0.05, + "adjustment_set": ["vaccine"], + "formula": "cum_vaccinations ~ cum_vaccinated + vaccine" + } + }, { + "name": "cum_vaccinations _||_ cum_infections", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "cum_vaccinations", "outcome_variable": "cum_infections", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_infections ~ cum_vaccinations + vaccine"}, "alpha": 0.05, - "skip": false + "adjustment_set": ["vaccine"], + "formula": "cum_infections ~ cum_vaccinations + vaccine" + } + }, { + "name": "cum_infections _||_ cum_vaccinations", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 }, - { - "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", - "estimator": "LinearRegressionEstimator", - "effect_measure": "coefficient", - "effect": "direct", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_infections", + "outcome_variable": "cum_vaccinations", + "alpha": 0.05, + "adjustment_set": ["vaccine"], + "formula": "cum_vaccinations ~ cum_infections + vaccine" + } + }, { + "name": "cum_vaccinated _||_ cum_infections", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", "treatment_variable": "cum_vaccinated", "outcome_variable": "cum_infections", - "expected_effect": {"name": "NoEffect"}, - "estimator_kwargs": {"formula": "cum_infections ~ cum_vaccinated + vaccine"}, "alpha": 0.05, - "skip": false + "adjustment_set": ["vaccine"], + "formula": "cum_infections ~ cum_vaccinated + vaccine" + } + }, { + "name": "cum_infections _||_ cum_vaccinated", + "skip": false, + "effect_measure": "coefficient", + "query": null, + "expected_effect": { + "name": "NoEffect", + "effect_type": "direct", + "atol": 0, + "ctol": 0.0 + }, + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "cum_infections", + "outcome_variable": "cum_vaccinated", + "alpha": 0.05, + "adjustment_set": ["vaccine"], + "formula": "cum_vaccinated ~ cum_infections + vaccine" } - ] + }] } diff --git a/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb b/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb index b67fe911..93e43d38 100644 --- a/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb +++ b/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb @@ -129,15 +129,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading DAG from dag.dot\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - DAG loaded with 5 nodes and 3 edges\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading data from 1 source(s)\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Initial data shape: (60, 16)\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading test configurations from causal_tests.json\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Running causal tests...\n", - "100%|████████████████████████████████████████████| 9/9 [00:00<00:00, 271.76it/s]\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Saving results to causal_test_results.json\n", - "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Results saved successfully\n" + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Loading DAG from dag.dot\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - DAG loaded with 5 nodes and 3 edges\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Loading data from 1 source(s)\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Initial data shape: (60, 16)\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Loading test configurations from causal_tests.json\n", + "2026-09-07 09:30:58 - causal_testing.causal_testing_framework - INFO - Running causal tests...\n", + " 0%| | 0/17 [00:00" + "
" ] }, "metadata": {}, @@ -190,6 +190,7 @@ "fig, ax = plt.subplots(nrows=len(causal_tests_results), ncols=1, figsize=[10,100])\n", "\n", "dag_copy = dag.copy()\n", + "dag_copy.ignore_cycles=True\n", "\n", "for i, test in enumerate(causal_tests_results):\n", " treatment_node = test['estimator']['treatment_variable']\n", diff --git a/pyproject.toml b/pyproject.toml index c33ced14..9fc006ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "pandas>=2.1", "scikit_learn~=1.4", "scipy>=1.12.0,<=1.17.1", - "statsmodels~=0.14", + "statsmodels~=0.15", "tabulate~=0.9", "pydot>=2.0", "pygad~=3.3", diff --git a/tests/discovery_tests/test_abstract_discovery.py b/tests/discovery_tests/test_abstract_discovery.py index 1721f400..eaf1c64b 100644 --- a/tests/discovery_tests/test_abstract_discovery.py +++ b/tests/discovery_tests/test_abstract_discovery.py @@ -274,7 +274,7 @@ def test_evaluate_tests_inestimable(self): "outcome": "completed", }, { - "result": TestOutcome.INESTIMABLE, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "completed", diff --git a/tests/estimation_tests/test_instrumental_variable_estimator.py b/tests/estimation_tests/test_instrumental_variable_estimator.py index a4e1edf3..10f72fca 100644 --- a/tests/estimation_tests/test_instrumental_variable_estimator.py +++ b/tests/estimation_tests/test_instrumental_variable_estimator.py @@ -25,8 +25,6 @@ def test_estimate_coefficient(self): iv_estimator = InstrumentalVariableEstimator( treatment_variable="X", outcome_variable="Y", - treatment_value=None, - control_value=None, instrument="Z", ) effect_estimate = iv_estimator.estimate_coefficient(self.df) @@ -38,8 +36,6 @@ def test_to_dict(self): iv_estimator = InstrumentalVariableEstimator( treatment_variable="X", outcome_variable="Y", - control_value=0, - treatment_value=1, instrument="Z", ) self.assertEqual( @@ -49,8 +45,6 @@ def test_to_dict(self): "treatment_variable": "X", "outcome_variable": "Y", "alpha": 0.05, - "control_value": 0, - "treatment_value": 1, "instrument": "Z", "bootstrap_size": 100, }, diff --git a/tests/main_tests/test_ctf.py b/tests/main_tests/test_ctf.py index 312d3acf..cf38563a 100644 --- a/tests/main_tests/test_ctf.py +++ b/tests/main_tests/test_ctf.py @@ -57,7 +57,7 @@ def test_create_test_case_invalid_estimator(self): "treatment_variable": "test_input", "outcome_variable": "test_output", "expected_effect": {"name": "NoEffect"}, - "estimator": "InvalidEstimator", + "estimator": {"name": "InvalidEstimator"}, } ) self.assertEqual( @@ -79,22 +79,45 @@ def test_create_test_case_no_estimator(self): } ) self.assertEqual( - "Test configuration must specify an estimator", + "Test configuration must specify an estimator.", str(e.exception), ) + def test_create_test_case_no_expected_effect(self): + framework = CausalTestingFramework() + framework.load_dag(self.dag_path) + framework.load_data(self.data_paths) + test = { + "name": "test1", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "adjustment_set": [], + }, + "effect_measure": "coefficient", + } + with self.assertRaises(ValueError) as e: + framework.create_causal_test(test) + self.assertEqual( + "Test configuration must specify an expected effect.", + str(e.exception), + ) + def test_create_test_case_invalid_effect(self): framework = CausalTestingFramework() framework.load_dag(self.dag_path) framework.load_data(self.data_paths) test = { "name": "test1", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "adjustment_set": [], + }, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "InvalidEffect"}, - "estimator_kwargs": {"adjustment_set": []}, } with self.assertRaises(ValueError) as e: framework.create_causal_test(test) @@ -111,12 +134,14 @@ def test_create_test_case_effect_kwargs(self): framework.load_data(self.data_paths) test = { "name": "test1", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", + "estimator": { + "name": "LinearRegressionEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "adjustment_set": [], + }, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "ExactValue", "value": 4}, - "estimator_kwargs": {"adjustment_set": []}, } test_case = framework.create_causal_test(test) self.assertEqual(test_case.expected_causal_effect.value, 4) @@ -127,12 +152,14 @@ def test_create_test_case_estimator_kwargs(self): framework.load_data(self.data_paths) test = { "name": "test1", - "treatment_variable": "test_input", - "estimator": "InstrumentalVariableEstimator", + "estimator": { + "name": "InstrumentalVariableEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "instrument": "instrumental_variable", + }, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "SomeEffect"}, - "estimator_kwargs": {"instrument": "instrumental_variable"}, } test_case = framework.create_causal_test(test) self.assertEqual(test_case.estimator.instrument, "instrumental_variable") @@ -197,14 +224,14 @@ def test_ctf_evaluate_dag_inestimable(self): expected = pd.Series( { "FAIL": 1, - "FAIL_ci_high": 2, + "FAIL_ci_high": 1, "FAIL_ci_low": 0, - "INESTIMABLE": 1, - "INESTIMABLE_ci_high": 1, + "INESTIMABLE": 0, + "INESTIMABLE_ci_high": 0, "INESTIMABLE_ci_low": 0, - "PASS": 4, - "PASS_ci_high": 4, - "PASS_ci_low": 0, + "PASS": 5, + "PASS_ci_high": 5, + "PASS_ci_low": 2, } ).sort_index() pd.testing.assert_series_equal(results, expected) diff --git a/tests/main_tests/test_main.py b/tests/main_tests/test_main.py index eddb4938..239abe44 100644 --- a/tests/main_tests/test_main.py +++ b/tests/main_tests/test_main.py @@ -110,6 +110,47 @@ def test_parse_args_bootstrap_size_explicit_adequacy(self): executed_tests = [test for test in log if not test.get("skip", False)] assert all(test["result"].get("bootstrap_size", 50) == 50 for test in executed_tests) + def test_parse_args_generate_and_test(self): + with tempfile.TemporaryDirectory() as tmp: + with patch( + "sys.argv", + [ + "causal_testing", + "generate", + "--dag-path", + str(self.dag_path), + "--data-paths", + str(self.data_paths[0]), + "--output", + os.path.join(tmp, "tests.json"), + ], + ): + main() + self.assertTrue(os.path.exists(os.path.join(tmp, "tests.json"))) + with patch( + "sys.argv", + [ + "causal_testing", + "test", + "--dag-path", + str(self.dag_path), + "--data-paths", + str(self.data_paths[0]), + "--test-config", + os.path.join(tmp, "tests.json"), + "--output", + str(self.output_path.parent / "main.json"), + "-A", + "-b", + "50", + ], + ): + main() + with open(self.output_path.parent / "main.json", encoding="utf-8") as f: + log = json.load(f) + executed_tests = [test for test in log if not test.get("skip", False)] + assert all(test["result"].get("bootstrap_size", 50) == 50 for test in executed_tests) + def test_parse_args_generation(self): with tempfile.TemporaryDirectory() as tmp: with patch( diff --git a/tests/resources/data/tests.json b/tests/resources/data/tests.json index 9ee0171f..d6f64e5b 100644 --- a/tests/resources/data/tests.json +++ b/tests/resources/data/tests.json @@ -1,23 +1,28 @@ { "tests": [{ "name": "test1", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", + "estimator": { + "name": "LinearRegressionEstimator", + "outcome_variable": "test_output", + "treatment_variable": "test_input", + "adjustment_set": [] + }, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "NoEffect"}, + "expected_effect": {"name": "NoEffect"}, "skip": false, - "query": "test_input > 0", - "estimator_kwargs": {"adjustment_set": []} - + "query": "test_input > 0" }, { "name": "test2", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", + "estimator": { + "name":"LinearRegressionEstimator", + "treatment_variable": "test_input", + "outcome_variable": "test_output", + "adjustment_set": [] + }, + "expected_effect": {"name": "NoEffect"}, "effect_measure": "coefficient", - "outcome_variable": "test_output", "expected_effect": {"name": "NoEffect"}, "skip": true, - "query": "test_input <= 5", - "estimator_kwargs": {"adjustment_set": []} + "query": "test_input <= 5" }] } diff --git a/tests/testing_tests/test_causal_test_adequacy.py b/tests/testing_tests/test_causal_test_adequacy.py index 2aec4a40..28980abd 100644 --- a/tests/testing_tests/test_causal_test_adequacy.py +++ b/tests/testing_tests/test_causal_test_adequacy.py @@ -210,7 +210,7 @@ def test_to_dict(self): # Use json_normalize to avoid rounding errors pd.testing.assert_frame_equal( pd.json_normalize(expected_dict).round(2), - pd.json_normalize(adequacy_metric.to_dict(include_results=True)).round(2), + pd.json_normalize(adequacy_metric.to_dict(include_adequacy_results=True)).round(2), ) def test_dag_adequacy_dependent(self):