diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index d4f94d52..fbeddcfc 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,3 +1,9 @@
+## Summary
+What does this PR do?
+
+## Main file changes
+Summarise changes to main files to be reviewed.
+
## Checklist
Before you mark your PR as ready for review, please ensure you have completed the following.
diff --git a/causal_testing/__main__.py b/causal_testing/__main__.py
index ddc9641b..d0f1ff95 100644
--- a/causal_testing/__main__.py
+++ b/causal_testing/__main__.py
@@ -13,6 +13,7 @@
from causal_testing.causal_testing_framework import CausalTestingFramework, read_dataframe
from causal_testing.specification.causal_dag import CausalDAG
+from causal_testing.visualisation.testing_dashboard import Dashboard
logger = logging.getLogger(__name__)
@@ -26,6 +27,7 @@ class Command(Enum):
GENERATE = "generate"
DISCOVER = "discover"
EVALUATE = "evaluate"
+ VISUALISE = "visualise"
def setup_logging(level: str) -> None:
@@ -79,10 +81,21 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace:
"-s",
"--silent",
action="store_true",
- help="Do not crash on error. If set to true, errors are recorded as test results.",
+ help="Do not crash on error. If set to true, errors are recorded as test results. (Defaults to False)",
+ default=False,
+ )
+ parser_test.add_argument(
+ "-R",
+ "--include-adequacy-results",
+ action="store_true",
+ help="Include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy "
+ "bootstraps. (Defaults to False)",
default=False,
)
+ # Visualisation
+ parser_visualise = subparsers.add_parser(Command.VISUALISE.value, help="Visualise causal test results")
+
# DAG evaluation
parser_evaluate = subparsers.add_parser(
Command.EVALUATE.value, help="Evaluate how well a causal DAG fits a dataset"
@@ -142,7 +155,7 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace:
default=[],
)
- for parser in [parser_generate, parser_discover, parser_test, parser_evaluate]:
+ for parser in [parser_generate, parser_discover, parser_test, parser_evaluate, parser_visualise]:
parser.add_argument(
"-l",
"--log_level",
@@ -151,6 +164,7 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace:
choices=["NONE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
help="Set the logging level (default: WARNING).",
)
+ for parser in [parser_generate, parser_discover, parser_test, parser_evaluate]:
parser.add_argument(
"-a",
"--alpha",
@@ -197,7 +211,7 @@ def main() -> None:
skip=False,
)
with open(args.output, "w", encoding="utf-8") as f:
- json.dump({"tests": [test.to_dict() for test in causal_tests]}, f)
+ json.dump([test.to_dict() for test in causal_tests], f)
logging.info("Causal test generation completed successfully.")
case Command.DISCOVER:
@@ -257,9 +271,12 @@ def main() -> None:
logging.info("Running tests")
framework.run_tests(silent=args.silent, adequacy=args.adequacy, bootstrap_size=args.bootstrap_size)
- framework.save_results(args.output)
+ framework.save_results(args.output, include_adequacy_results=args.include_adequacy_results)
logging.info("Causal testing completed successfully.")
+ case Command.VISUALISE:
+ dashboard = Dashboard()
+ dashboard.serve()
case Command.EVALUATE:
# Create and setup framework
framework = CausalTestingFramework()
diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py
index 016b1d61..6b83bd70 100644
--- a/causal_testing/causal_testing_framework.py
+++ b/causal_testing/causal_testing_framework.py
@@ -11,40 +11,44 @@
import pandas as pd
from tqdm import tqdm
+from causal_testing.estimation.effect_estimate import EffectEstimate
from causal_testing.specification.causal_dag import CausalDAG
from causal_testing.testing.causal_test_case import CausalTestCase
-from causal_testing.testing.causal_test_result import TestOutcome
+from causal_testing.testing.causal_test_result import CausalTestResult, TestOutcome
+from causal_testing.testing.data_adequacy import DataAdequacy
logger = logging.getLogger(__name__)
-
-def read_dataframe(file_path: str, **kwargs: dict) -> pd.DataFrame:
+data_readers = {
+ ".csv": pd.read_csv,
+ ".xlsx": pd.read_excel,
+ ".xls": pd.read_excel,
+ ".html": pd.read_html,
+ ".xml": pd.read_xml,
+ ".feather": pd.read_feather,
+ ".parquet": pd.read_parquet,
+ ".pq": pd.read_parquet,
+ ".pqt": pd.read_parquet,
+ ".json": pd.read_json,
+ ".stata": pd.read_stata,
+}
+
+
+def read_dataframe(file_path: str, content: bytes = None, **kwargs: dict) -> pd.DataFrame:
"""
Read data into a dataframe.
:param file_path: The path to the data.
+ :param content: The bytes content of the file.
:param kwargs: Keyword arguments to be passed to the `read_` function.
:returns: The read-in DataFrame.
"""
- readers = {
- ".csv": pd.read_csv,
- ".xlsx": pd.read_excel,
- ".xls": pd.read_excel,
- ".html": pd.read_html,
- ".xml": pd.read_xml,
- ".feather": pd.read_feather,
- ".parquet": pd.read_parquet,
- ".pq": pd.read_parquet,
- ".pqt": pd.read_parquet,
- ".json": pd.read_json,
- ".stata": pd.read_stata,
- }
suffix = Path(file_path).suffix.lower()
- if suffix in readers:
- return readers[suffix](file_path, **kwargs)
+ if suffix in data_readers:
+ return data_readers[suffix](content if content is not None else file_path, **kwargs)
raise ValueError(f"Unsupported file extension: '{suffix}'")
@@ -61,9 +65,9 @@ def __init__(self, dag: CausalDAG = None, test_cases: list[CausalTestCase] = Non
def setup(
self,
- dag_path: str,
- data_paths: list[str],
- test_cases_path: str,
+ dag_path: str = None,
+ data_paths: list[str] = None,
+ test_cases_path: str = None,
ignore_cycles: bool = False,
query: str = None,
**kwargs: dict,
@@ -78,9 +82,12 @@ def setup(
:param query: Optional pandas query string to filter the loaded data
:param kwargs: Keyword arguments to be passed to the `read_` function.
"""
- self.load_dag(dag_path, ignore_cycles)
- self.load_data(data_paths, query, **kwargs)
- self.load_test_cases_from_json(test_cases_path)
+ if dag_path is not None:
+ self.load_dag(dag_path, ignore_cycles)
+ if data_paths is not None:
+ self.load_data(data_paths, query, **kwargs)
+ if test_cases_path is not None:
+ self.load_test_cases_from_json(test_cases_path)
def load_dag(self, dag_path: str, ignore_cycles: bool = False):
"""
@@ -120,21 +127,13 @@ def load_test_cases_from_json(self, test_cases_path: str):
"""
logger.info(f"Loading test configurations from {test_cases_path}")
- if self.dag is None or self.df is None:
- raise ValueError("Please load DAG and data before attempting to load tests.")
+ if self.dag is None:
+ raise ValueError("Please load DAG before attempting to load tests.")
with open(test_cases_path, "r", encoding="utf-8") as f:
test_configs = json.load(f)
- test_cases = []
-
- for test in test_configs.get("tests", []):
-
- # Create causal test case
- causal_test = self.create_causal_test(test)
- test_cases.append(causal_test)
-
- self.test_cases = test_cases
+ self.test_cases = [self.create_causal_test(test) for test in test_configs]
def create_causal_test(self, test: dict) -> CausalTestCase:
"""
@@ -145,54 +144,51 @@ def create_causal_test(self, test: dict) -> CausalTestCase:
:return: CausalTestCase object
:raises: ValueError if invalid estimator or configuration is provided
"""
+ # Create the estimator with correct parameters
estimator_map = {ff.name: ff for ff in entry_points(group="estimators")}
- effect_map = {ff.name: ff for ff in entry_points(group="causal_effects")}
-
if "estimator" not in test:
- raise ValueError("Test configuration must specify an estimator")
-
- if test["estimator"] not in estimator_map:
+ raise ValueError("Test configuration must specify an `estimator`.")
+ estimator_kwargs = test["estimator"]
+ estimator_name = estimator_kwargs.pop("name")
+ if estimator_name not in estimator_map:
raise ValueError(
- f"Unsupported estimator {test['estimator']}. Supported: {sorted(estimator_map)}. "
+ f"Unsupported estimator {estimator_name}. Supported: {sorted(estimator_map)}. "
"If you have implemented a custom estimator, you will need to add this to your entrypoints via your "
"pyproject.toml file."
)
+ test["estimator"] = estimator_map.get(estimator_name).load()(**estimator_kwargs)
- # Create the estimator with correct parameters
- treatment_variable = test.get("treatment_variable")
- outcome_variable = test.get("outcome_variable")
- estimator_class = estimator_map.get(test["estimator"]).load()
- estimator_kwargs = test.get("estimator_kwargs", {})
- effect_type = test.get("expected_effect", {}).get("effect_type", "direct")
-
- estimator = estimator_class(
- treatment_variable=treatment_variable,
- outcome_variable=outcome_variable,
- treatment_value=test.get("treatment_value"),
- control_value=test.get("control_value"),
- alpha=test.get("alpha", 0.05),
- **estimator_kwargs,
- )
-
- # Get effect type and create expected effect
- expected_effect = test["expected_effect"]
- effect_type = expected_effect.pop("name")
- if effect_type not in effect_map:
+ # Create an effect with the corect parameters
+ effect_map = {ff.name: ff for ff in entry_points(group="causal_effects")}
+ if "expected_causal_effect" not in test:
+ raise ValueError("Test configuration must specify an `expected_causal_effect`.")
+ expected_causal_effect_kwargs = test["expected_causal_effect"]
+ expected_causal_effect_name = expected_causal_effect_kwargs.pop("name")
+ if expected_causal_effect_name not in effect_map:
raise ValueError(
- f"Unsupported causal effect {effect_type}. Supported: {sorted(effect_map)}. "
+ f"Unsupported causal effect {expected_causal_effect_name}. Supported: {sorted(effect_map)}. "
"If you have implemented a custom causal effect, you will need to add this to your entrypoints via "
"your pyproject.toml file."
)
- expected_effect = effect_map[effect_type].load()(**expected_effect)
+ test["expected_causal_effect"] = effect_map[expected_causal_effect_name].load()(**expected_causal_effect_kwargs)
+
+ if "result" in test:
+ outcome = getattr(TestOutcome, test["result"]["outcome"]) if "outcome" in test["result"] else None
+ effect_estimate = (
+ EffectEstimate(**test["result"]["effect_estimate"]) if "effect_estimate" in test["result"] else None
+ )
+ adequacy = DataAdequacy(**test["result"]["adequacy"]) if "adequacy" in test["result"] else None
+
+ test["result"] = CausalTestResult(outcome=outcome, effect_estimate=effect_estimate, adequacy=adequacy)
+
+ return CausalTestCase(**test)
- return CausalTestCase(
- name=test.get("name"),
- effect_measure=test.get("effect_measure"),
- query=test.get("query"),
- expected_causal_effect=expected_effect,
- estimator=estimator,
- skip=test.get("skip", False),
- )
+ def ready_to_run(self) -> bool:
+ """
+ Test whether framework is ready to run test cases.
+ :returns: True if the DAG, data, and test cases are defined.
+ """
+ return all(x is not None for x in (self.test_cases, self.dag, self.df)) and bool(self.test_cases)
def run_tests(self, silent: bool = False, adequacy: bool = False, bootstrap_size: int = 100):
"""
@@ -267,8 +263,14 @@ def evaluate_dag(self, bootstrap_size: bool = 100, alpha: float = 0.05) -> pd.Se
return pd.Series(results).sort_index()
- def save_results(self, output_path) -> list:
- """Save test results to JSON file in the expected format."""
+ def save_results(self, output_path: str, include_adequacy_results: bool = False):
+ """
+ Save test results to JSON file in the expected format.
+
+ :param output_path: Path for output file (.json).
+ :param include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy
+ bootstraps.
+ """
logger.info(f"Saving results to {output_path}")
# Create parent directory if it doesn't exist
@@ -276,6 +278,17 @@ def save_results(self, output_path) -> list:
# Save to file
with open(output_path, "w", encoding="utf-8") as f:
- json.dump([test.to_dict() for test in self.test_cases], f, indent=2)
+ json.dump(
+ [test.to_dict(include_adequacy_results=include_adequacy_results) for test in self.test_cases],
+ f,
+ indent=2,
+ )
logger.info("Results saved successfully")
+
+ def test_dataframe(self) -> pd.DataFrame:
+ """
+ :returns: The causal test cases as a dataframe. Nested objects such as results are indexed as, e.g.
+ `result.outcome`.
+ """
+ return pd.json_normalize(map(lambda t: t.to_dict(), self.test_cases))
diff --git a/causal_testing/discovery/abstract_discovery.py b/causal_testing/discovery/abstract_discovery.py
index c0b51387..0ba89871 100644
--- a/causal_testing/discovery/abstract_discovery.py
+++ b/causal_testing/discovery/abstract_discovery.py
@@ -8,15 +8,12 @@
from abc import ABC, abstractmethod
from itertools import permutations
-import networkx as nx
import numpy as np
import pandas as pd
import rustworkx as rx
from causal_testing.causal_testing_framework import CausalTestingFramework
from causal_testing.specification.causal_dag import CausalDAG
-from causal_testing.testing.causal_effect import Negative, Positive
-from causal_testing.testing.causal_test_case import CausalTestCase
from causal_testing.testing.causal_test_result import TestOutcome
# Ignore warnings from statsmodels when we try to evaluate test cases
@@ -97,22 +94,6 @@ def discover(self) -> CausalDAG:
:returns: The inferred causal DAG.
"""
- def effect_direction(self, test_case: CausalTestCase) -> str:
- """
- Check whether the estimated causal effect is negative or positive.
-
- :param test_case: The causal test case.
- :returns: Whether the estimated causal test is positive or negative (or no effect).
- """
- if pd.api.types.is_numeric_dtype(self.df[test_case.treatment_variable]) and pd.api.types.is_numeric_dtype(
- self.df[test_case.outcome_variable]
- ):
- if Negative().apply(test_case.result.effect_estimate):
- return "negative"
- if Positive().apply(test_case.result.effect_estimate):
- return "positive"
- return None
-
def remove_cycles(self, causal_dag: CausalDAG):
"""
Remove cycles from individuals by iteratively deleting a random edge from each cycle until there are no more
@@ -130,52 +111,6 @@ def remove_cycles(self, causal_dag: CausalDAG):
cycle = simple_cycle(causal_dag)
causal_dag.add_nodes_from(nodes)
- def write_dot(self, individual: CausalDAG, output_file: str):
- """
- Write the given individual to the given output file.
-
- :param individual: The causal DAG to output.
- :param output_file: The name of the file to write to.
- """
- if hasattr(individual, "test_results"):
- for _, test in individual.test_results.iterrows():
- if (test["treatment"], test["outcome"]) in individual.edges:
- individual[test["treatment"]][test["outcome"]]["label"] = test["effect"]
-
- print(test)
-
- if test["result"] == TestOutcome.PASS:
- print(" GREEN")
- individual[test["treatment"]][test["outcome"]]["color"] = "green"
- individual[test["treatment"]][test["outcome"]]["fontcolor"] = "green"
- elif test["result"] == TestOutcome.INESTIMABLE:
- print(" ORANGE")
- individual[test["treatment"]][test["outcome"]]["color"] = "orange"
- individual[test["treatment"]][test["outcome"]]["fontcolor"] = "orange"
- elif test["result"] == TestOutcome.FAIL:
- print(" RED")
- individual[test["treatment"]][test["outcome"]]["color"] = "red"
- individual[test["treatment"]][test["outcome"]]["fontcolor"] = "red"
- else:
- raise ValueError(f"Invalid test outcome {test['result']}")
- else:
- individual.add_edge(test["treatment"], test["outcome"], ignore_cycles=True)
- individual[test["treatment"]][test["outcome"]]["style"] = "dashed"
- individual[test["treatment"]][test["outcome"]]["label"] = test["effect"]
- if test["result"] == TestOutcome.PASS:
- individual[test["treatment"]][test["outcome"]]["style"] = "invis"
- individual[test["treatment"]][test["outcome"]]["constraint"] = False
- elif test["result"] == TestOutcome.INESTIMABLE:
- individual[test["treatment"]][test["outcome"]]["color"] = "orange"
- individual[test["treatment"]][test["outcome"]]["fontcolor"] = "orange"
- elif test["result"] == TestOutcome.FAIL:
- individual[test["treatment"]][test["outcome"]]["color"] = "red"
- individual[test["treatment"]][test["outcome"]]["fontcolor"] = "red"
- else:
- raise ValueError(f"Invalid test outcome {test['result']}")
-
- nx.drawing.nx_pydot.write_dot(individual, output_file)
-
def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame:
"""
Generate and evaluate causal test cases from the supplied CausalDAG and return a list of edges for which the
@@ -190,6 +125,7 @@ def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame:
ctf = CausalTestingFramework(dag=causal_dag, df=self.df)
causal_dag.datatypes = self.df.dtypes
ctf.test_cases = causal_dag.generate_causal_tests()
+ causal_dag.test_cases = ctf.test_cases
results = []
@@ -206,7 +142,6 @@ def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame:
"expected_effect": test_case.expected_causal_effect.__class__.__name__,
"treatment": test_case.treatment_variable,
"outcome": test_case.outcome_variable,
- "effect": self.effect_direction(test_case),
}
)
except np.linalg.LinAlgError:
@@ -219,7 +154,4 @@ def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame:
}
)
- causal_dag.test_results = pd.DataFrame(results)
-
- results = pd.DataFrame(results)
return pd.DataFrame(results)
diff --git a/causal_testing/discovery/hill_climber_discovery.py b/causal_testing/discovery/hill_climber_discovery.py
index b66a6be6..cf1cac30 100644
--- a/causal_testing/discovery/hill_climber_discovery.py
+++ b/causal_testing/discovery/hill_climber_discovery.py
@@ -75,16 +75,14 @@ def evaluate_fitness(
:returns: Tuple of the form (X, Y), where X is a triple containing the number of passing, failing, and
inestimable tests respectively, and Y is a list of failing edges.
"""
- self.evaluate_tests(individual)
- counts = self.sum_test_outcomes(individual.test_results)
+ test_results = self.evaluate_tests(individual)
+ counts = self.sum_test_outcomes(test_results)
# Add extra "var1" and "var2" columns to serve as order independent "treatment" and "outcome"
query_df = pd.concat(
[
- individual.test_results,
- pd.DataFrame(
- np.sort(individual.test_results[["treatment", "outcome"]], axis=1), columns=["var1", "var2"]
- ),
+ test_results,
+ pd.DataFrame(np.sort(test_results[["treatment", "outcome"]], axis=1), columns=["var1", "var2"]),
],
axis=1,
)
diff --git a/causal_testing/estimation/effect_estimate.py b/causal_testing/estimation/effect_estimate.py
index 865ecd61..f856a6af 100644
--- a/causal_testing/estimation/effect_estimate.py
+++ b/causal_testing/estimation/effect_estimate.py
@@ -19,10 +19,13 @@ class EffectEstimate:
:ivar ci_high: The upper confidence interval
"""
- type: str
- value: pd.Series
- ci_low: pd.Series = None
- ci_high: pd.Series = None
+ def __init__(
+ self, effect_measure: str, effect_estimate: pd.Series, ci_low: pd.Series = None, ci_high: pd.Series = None
+ ):
+ self.effect_measure = effect_measure
+ self.effect_estimate = pd.Series(effect_estimate)
+ self.ci_low = pd.Series(ci_low) if ci_low is not None else None
+ self.ci_high = pd.Series(ci_high) if ci_high is not None else None
def ci_valid(self) -> bool:
"""Return whether or not the result has valid confidence invervals"""
@@ -34,11 +37,14 @@ def ci_valid(self) -> bool:
def to_dict(self) -> dict:
"""Return representation as a dict."""
- d = {"effect_measure": self.type, "effect_estimate": self.value.to_dict()}
+ d = {
+ "effect_measure": self.effect_measure,
+ "effect_estimate": self.effect_estimate.to_dict(),
+ }
if self.ci_valid():
return d | {"ci_low": self.ci_low.to_dict(), "ci_high": self.ci_high.to_dict()}
return d
def to_df(self) -> pd.DataFrame:
"""Return representation as a pandas dataframe."""
- return pd.DataFrame({"effect_estimate": self.value, "ci_low": self.ci_low, "ci_high": self.ci_high})
+ return pd.DataFrame({"effect_estimate": self.effect_estimate, "ci_low": self.ci_low, "ci_high": self.ci_high})
diff --git a/causal_testing/estimation/instrumental_variable_estimator.py b/causal_testing/estimation/instrumental_variable_estimator.py
index f4ffab37..e52515dc 100644
--- a/causal_testing/estimation/instrumental_variable_estimator.py
+++ b/causal_testing/estimation/instrumental_variable_estimator.py
@@ -25,8 +25,6 @@ def __init__(
self,
outcome_variable: str,
treatment_variable: str,
- treatment_value: float,
- control_value: float,
instrument: str,
alpha: float = 0.05,
bootstrap_size=100,
@@ -34,8 +32,6 @@ def __init__(
super().__init__(
treatment_variable=treatment_variable,
outcome_variable=outcome_variable,
- treatment_value=treatment_value,
- control_value=control_value,
alpha=alpha,
)
@@ -47,13 +43,17 @@ def add_modelling_assumptions(self):
Add modelling assumptions to the estimator. This is a list of strings which list the modelling assumptions that
must hold if the resulting causal inference is to be considered valid.
"""
- self.modelling_assumptions.append("""The instrument and the treatment, and the treatment and the outcome must be
- related linearly in the form Y = aX + b.""")
- self.modelling_assumptions.append("""The three IV conditions must hold
+ self.modelling_assumptions.append(
+ """The instrument and the treatment, and the treatment and the outcome must be
+ related linearly in the form Y = aX + b."""
+ )
+ self.modelling_assumptions.append(
+ """The three IV conditions must hold
(i) Instrument is associated with treatment
(ii) Instrument does not affect outcome except through its potential effect on treatment
(iii) Instrument and outcome do not share causes
- """)
+ """
+ )
def iv_coefficient(self, df) -> float:
"""
diff --git a/causal_testing/estimation/linear_regression_estimator.py b/causal_testing/estimation/linear_regression_estimator.py
index deba16ce..2c91ded7 100644
--- a/causal_testing/estimation/linear_regression_estimator.py
+++ b/causal_testing/estimation/linear_regression_estimator.py
@@ -155,7 +155,7 @@ def estimate_ate_calculated(self, df: pd.DataFrame) -> EffectEstimate:
return EffectEstimate("ate", pd.Series(treatment_outcome["mean"] - control_outcome["mean"]), ci_low, ci_high)
def _get_confidence_intervals(self, model, treatment):
- confidence_intervals = model.conf_int(alpha=self.alpha, cols=None)
+ confidence_intervals = model.conf_int(alpha=self.alpha)
ci_low, ci_high = (
pd.Series(confidence_intervals[0].loc[treatment]),
pd.Series(confidence_intervals[1].loc[treatment]),
diff --git a/causal_testing/testing/causal_effect.py b/causal_testing/testing/causal_effect.py
index a99a3bd1..834c7c63 100644
--- a/causal_testing/testing/causal_effect.py
+++ b/causal_testing/testing/causal_effect.py
@@ -38,14 +38,14 @@ class SomeEffect(CausalEffect):
"""An extension of CausalEffect representing that the expected causal effect should not be zero."""
def apply(self, effect_estimate: EffectEstimate) -> bool:
- if effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio", "odds_ratio"):
+ if effect_estimate.effect_measure in ("risk_ratio", "hazard_ratio", "unit_odds_ratio", "odds_ratio"):
value_to_check = 1
- elif effect_estimate.type in ("coefficient", "ate"):
+ elif effect_estimate.effect_measure in ("coefficient", "ate"):
value_to_check = 0
else:
- raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect")
+ raise ValueError(f"Test Value type {effect_estimate.effect_measure} is not valid for this CausalEffect")
- return (~((effect_estimate.ci_low <= value_to_check) & (value_to_check <= effect_estimate.ci_high))).all()
+ return (~((effect_estimate.ci_low <= value_to_check) & (value_to_check <= effect_estimate.ci_high))).any()
class NoEffect(CausalEffect):
@@ -62,17 +62,17 @@ def __init__(self, effect_type: str = "direct", atol: float = 0, ctol: float = 0
self.ctol = ctol
def apply(self, effect_estimate: EffectEstimate) -> bool:
- if effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio", "odds_ratio"):
+ if effect_estimate.effect_measure in ("risk_ratio", "hazard_ratio", "unit_odds_ratio", "odds_ratio"):
value_to_check = 1
- elif effect_estimate.type in ("coefficient", "ate"):
+ elif effect_estimate.effect_measure in ("coefficient", "ate"):
value_to_check = 0
else:
- raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect")
+ raise ValueError(f"Test Value type {effect_estimate.effect_measure} is not valid for this CausalEffect")
return sum(
((effect_estimate.ci_low <= value_to_check) & (value_to_check <= effect_estimate.ci_high))
- | (np.isclose(effect_estimate.value, value_to_check, atol=self.atol))
- ) / len(effect_estimate.value) >= (1 - self.ctol)
+ | (np.isclose(effect_estimate.effect_estimate, value_to_check, atol=self.atol))
+ ) / len(effect_estimate.effect_estimate) >= (1 - self.ctol)
def to_dict(self):
"""
@@ -110,7 +110,7 @@ def __init__(
)
def apply(self, effect_estimate: EffectEstimate) -> bool:
- close = np.isclose(effect_estimate.value, self.value, atol=self.atol)
+ close = np.isclose(effect_estimate.effect_estimate, self.value, atol=self.atol)
if effect_estimate.ci_valid and self.ci_low is not None and self.ci_high is not None:
return (
close.all()
@@ -142,13 +142,13 @@ class Positive(SomeEffect):
Currently only single values are supported for the test value"""
def apply(self, effect_estimate: EffectEstimate) -> bool:
- if len(effect_estimate.value) > 1:
+ if len(effect_estimate.effect_estimate) > 1:
raise ValueError("Positive Effects are currently only supported on single float datatypes")
- if effect_estimate.type in {"ate", "coefficient"}:
+ if effect_estimate.effect_measure in {"ate", "coefficient"}:
return any(0 < ci_low < ci_high for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high))
- if effect_estimate.type in ["risk_ratio", "unit_odds_ratio"]:
+ if effect_estimate.effect_measure in ["risk_ratio", "unit_odds_ratio"]:
return any(1 < ci_low < ci_high for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high))
- raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect")
+ raise ValueError(f"Test Value type {effect_estimate.effect_measure} is not valid for this CausalEffect")
class Negative(SomeEffect):
@@ -156,11 +156,11 @@ class Negative(SomeEffect):
Currently only single values are supported for the test value"""
def apply(self, effect_estimate: EffectEstimate) -> bool:
- if len(effect_estimate.value) > 1:
+ if len(effect_estimate.effect_estimate) > 1:
raise ValueError("Negative Effects are currently only supported on single float datatypes")
- if effect_estimate.type in {"ate", "coefficient"}:
+ if effect_estimate.effect_measure in {"ate", "coefficient"}:
return any(ci_low < ci_high < 0 for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high))
- if effect_estimate.type in ["risk_ratio", "unit_odds_ratio"]:
+ if effect_estimate.effect_measure in ["risk_ratio", "unit_odds_ratio"]:
return any(ci_low < ci_high < 1 for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high))
# Dead code but necessary for pylint
- raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect")
+ raise ValueError(f"Test Value type {effect_estimate.effect_measure} is not valid for this CausalEffect")
diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py
index ec1de7b8..7ffe8ef1 100644
--- a/causal_testing/testing/causal_test_case.py
+++ b/causal_testing/testing/causal_test_case.py
@@ -36,11 +36,12 @@ def __init__(
name: str = None,
query: str = None,
skip: bool = False,
+ result: CausalTestResult = None,
):
self.expected_causal_effect = expected_causal_effect
self.effect_measure = effect_measure
self.estimator = estimator
- self.result = None
+ self.result = result
self.name = name
self.query = query
self.skip = skip
@@ -172,10 +173,11 @@ def estimate_effect(self, df: pd.DataFrame) -> CausalTestResult:
estimate_effect = getattr(self.estimator, f"estimate_{self.effect_measure}")
return estimate_effect(df)
- def to_dict(self) -> dict:
+ def to_dict(self, include_adequacy_results: bool = False) -> dict:
"""
Convert the test case to a python dictionary for easy serialisation as JSON.
+ :bool include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy bootstraps.
:returns: A JSON serialisable dict representing the test case.
"""
test_case = {
@@ -185,12 +187,11 @@ def to_dict(self) -> dict:
"query": self.query,
}
- for label, attribute in [
- ("expected_effect", self.expected_causal_effect),
- ("estimator", self.estimator),
- ("result", self.result),
- ]:
- if attribute is not None:
- test_case[label] = attribute.to_dict()
+ if self.expected_causal_effect is not None:
+ test_case["expected_causal_effect"] = self.expected_causal_effect.to_dict()
+ if self.estimator is not None:
+ test_case["estimator"] = self.estimator.to_dict()
+ if self.result is not None:
+ test_case["result"] = self.result.to_dict(include_adequacy_results=include_adequacy_results)
return test_case
diff --git a/causal_testing/testing/causal_test_result.py b/causal_testing/testing/causal_test_result.py
index 5bfb15c1..89121056 100644
--- a/causal_testing/testing/causal_test_result.py
+++ b/causal_testing/testing/causal_test_result.py
@@ -4,6 +4,7 @@
from enum import Enum
from causal_testing.estimation.effect_estimate import EffectEstimate
+from causal_testing.testing.causal_effect import Negative, Positive
TestOutcome = Enum("TestOutcome", [("PASS", 2), ("FAIL", 0), ("INESTIMABLE", 1)])
@@ -34,10 +35,12 @@ def passed(self) -> bool:
"""
return self.outcome == TestOutcome.PASS
- def to_dict(self):
+ def to_dict(self, include_adequacy_results: bool = False):
"""
Convert the result to a python dictionary for easy serialisation as JSON.
+ :param include_adequacy_results: Whether to include the effect estimate and test outcome for adequacy
+ bootstraps.
:returns: A JSON serialisable dict representing the test result.
"""
@@ -47,6 +50,21 @@ def to_dict(self):
effect_estimate = self.effect_estimate.to_dict() if self.effect_estimate else {}
- adequacy = self.adequacy.to_dict() if self.adequacy else {}
+ adequacy = self.adequacy.to_dict(include_adequacy_results=include_adequacy_results) if self.adequacy else {}
- return outcome | effect_estimate | {"adequacy": adequacy}
+ return outcome | {"effect_estimate": effect_estimate, "adequacy": adequacy}
+
+ def effect_direction(self) -> str:
+ """
+ Check whether the estimated causal effect is negative or positive.
+
+ :returns: Whether the estimated causal effect is positive or negative (or no effect).
+ """
+ if len(self.effect_estimate.effect_estimate) > 1:
+ # Don't bother checking categorical estimates since they're not numeric
+ return "categorical"
+ if Negative().apply(self.effect_estimate):
+ return "negative"
+ if Positive().apply(self.effect_estimate):
+ return "positive"
+ return "no effect"
diff --git a/causal_testing/testing/data_adequacy.py b/causal_testing/testing/data_adequacy.py
index 192ef1a1..b7ef640b 100644
--- a/causal_testing/testing/data_adequacy.py
+++ b/causal_testing/testing/data_adequacy.py
@@ -28,23 +28,23 @@ def __init__(
successful: int = None,
bootstrap_size: int = None,
):
- self.kurtosis = kurtosis
+ self.kurtosis = Series(kurtosis) if kurtosis is not None else None
self.passing = passing
self.results = results
self.successful = successful
self.bootstrap_size = bootstrap_size
- def to_dict(self, include_results: bool = False):
+ def to_dict(self, include_adequacy_results: bool = False):
"""
:returns: the adequacy object as a dictionary.
- :param include_results: Whether to serialise the results.
+ :param include_adequacy_results: Whether to serialise the results.
"""
result = {
- "kurtosis": self.kurtosis.to_dict(),
+ "kurtosis": self.kurtosis.to_dict() if self.kurtosis is not None else None,
"passing": self.passing,
"successful": self.successful,
"bootstrap_size": self.bootstrap_size,
}
- if include_results:
+ if include_adequacy_results:
return result | {"results": self.results.reset_index(drop=True).to_dict()}
return result
diff --git a/causal_testing/visualisation/geometry.py b/causal_testing/visualisation/geometry.py
new file mode 100644
index 00000000..67b2414a
--- /dev/null
+++ b/causal_testing/visualisation/geometry.py
@@ -0,0 +1,197 @@
+"""
+This module implements various helper functions for plotting geometry.
+"""
+
+import re
+
+import holoviews as hv
+import numpy as np
+import pandas as pd
+from bokeh.models import Arrow, Ellipse, NormalHead
+from holoviews.plotting.bokeh.graphs import GraphPlot
+from scipy.interpolate import make_splprep # pylint: disable=E0611
+
+
+def _get_split_category_order(df: pd.DataFrame, col: str, value_col: str) -> list:
+ """
+ Partitions categories into two groups relative to overall_median:
+ - Group median < overall_median: sorted by category min (ascending).
+ - Group median >= overall_median: sorted by category max (ascending).
+ """
+ stats = df.groupby(col)[value_col].agg(["median", "min", "max"]).reset_index()
+
+ # Sort lower half by min value, upper half by max value
+ lower_order = stats[stats["median"] < stats["median"].median()].sort_values(by="min", ascending=True)[col].tolist()
+ upper_order = stats[stats["median"] >= stats["median"].median()].sort_values(by="max", ascending=True)[col].tolist()
+
+ return lower_order + upper_order
+
+
+def sort_df_by_median_split(
+ df: pd.DataFrame,
+ value_col: str,
+ treatment_col: str = "estimator.treatment_variable",
+ outcome_col: str = "estimator.outcome_variable",
+ vdims: list[str] = None,
+) -> pd.DataFrame:
+ """
+ Sorts treatment and outcome variables relative to the overall median kurtosis.
+ """
+
+ vdims = [] if vdims is None else vdims
+
+ # Fill missing (treatment, outcome) combinations with empty rows
+ # We need this to ensure that it's possible to obtain the correct ordering in the heatmap
+ df = (
+ df.set_index([treatment_col, outcome_col])
+ .reindex(
+ pd.MultiIndex.from_product(
+ [
+ df[treatment_col].dropna().unique(),
+ df[outcome_col].dropna().unique(),
+ ],
+ names=[treatment_col, outcome_col],
+ )
+ )
+ .reset_index()
+ )
+
+ # Apply ordered categoricals so HoloViews maps the axes to these index positions
+ df_sorted = df[[treatment_col, outcome_col, value_col] + vdims].copy()
+ df_sorted[treatment_col] = pd.Categorical(
+ df[treatment_col], categories=_get_split_category_order(df, treatment_col, value_col), ordered=True
+ )
+ df_sorted[outcome_col] = pd.Categorical(
+ df[outcome_col], categories=_get_split_category_order(df, outcome_col, value_col), ordered=True
+ )
+
+ df_sorted = df_sorted.sort_values(by=[treatment_col, outcome_col]).dropna()
+
+ # Need to convert the values back to strings, otherwise holoviz thinks they're not unique
+ df_sorted[treatment_col] = df_sorted[treatment_col].astype(str)
+ df_sorted[outcome_col] = df_sorted[outcome_col].astype(str)
+ return df_sorted
+
+
+def parse_dot_spline(pos_str: str) -> list[tuple[float, float]]:
+ """
+ Parse Graphviz 'pos' string into control points.
+ See https://graphviz.org/docs/attr-types/splineType for syntax details.
+ NOTE: This will ignore segments separated by ";", but this shouldn't be a problem in our limited context.
+
+ :param pos_str: The graphviz position string representing the list of control points.
+ """
+ end_point = None
+ points = []
+
+ for point_type, x, y in re.findall(r"(?:(s|e),)?(\d+(?:.\d+)?),(\d+(?:\.\d+)?)", pos_str):
+ x, y = float(x), float(y)
+ if point_type == "s":
+ points = [(x, y)] + points
+ elif point_type == "e":
+ end_point = x, y
+ else:
+ points.append((x, y))
+
+ if end_point:
+ points.append(end_point)
+
+ # Remove consecutive duplicate points
+ points = np.array(points)
+ mask = np.ones(len(points), dtype=bool)
+ mask[1:] = np.any(np.diff(points, axis=0) != 0, axis=1)
+ return points[mask]
+
+
+def edge_spline(
+ dot_pos: str,
+ target_node_centre: tuple[float, float],
+ target_node_width: float,
+ target_node_height: float = 16,
+ num_points: int = 100,
+ shorten: float = 0.05,
+):
+ """
+ Generate an edge spline from the given control points, trimmed at source & target ellipse boundaries.
+
+ :param dot_pos: The DOT `pos` attribute of the edge, representing the control points of the spline.
+ :param target_node_centre: The coordinates of the centre of the target node.
+ :param target_node_width: The width of the target_node.
+ :param target_node_height: The height of the target_node (defaults to 16).
+ :param num_points: The number of spline points to generage (defaults to 100).
+ :param shorten: The percentage of the line to shorten by to allow for the arrow head (defaults to 5%).
+ """
+ b_spline, _ = make_splprep(parse_dot_spline(dot_pos).T, k=3, s=0)
+
+ # Evaluate the BSpline object at uniform parametric points
+ smooth_points = b_spline(np.linspace(0, 1, num_points)).T
+
+ # Calculate angle and boundary radius for end node
+ dx_e = target_node_centre[0] - smooth_points[-round(num_points * shorten)][0]
+ dy_e = target_node_centre[1] - smooth_points[-round(num_points * shorten)][1]
+ angle_end = np.arctan2(dy_e, dx_e)
+ r_end = (target_node_width * target_node_height) / np.sqrt(
+ (target_node_width * np.sin(angle_end)) ** 2 + (target_node_height * np.cos(angle_end)) ** 2
+ )
+
+ dists_end = np.hypot(smooth_points[:, 0] - target_node_centre[0], smooth_points[:, 1] - target_node_centre[1])
+ end_idx = len(smooth_points) - np.searchsorted(dists_end[::-1], r_end)
+
+ trimmed_path = smooth_points[:end_idx]
+ return trimmed_path
+
+
+def node_width(label: str, text_font_size: int = 9, padding: int = 24) -> float:
+ """
+ Calculate the width that a node should be to accomadate the label.
+
+ :param label: The node label.
+ :param text_font_size: The font size in pt.
+ :param padding: Node inner padding in pt.
+ """
+ return len(label) * text_font_size + padding
+
+
+def style_graph_hook(plot: GraphPlot, element: hv.Graph):
+ """
+ Hook to properly style nodes to be an ellipse of the correct size.
+
+ :param plot: The current plot figure.
+ :param element: The Graph element.
+ """
+ fig = plot.handles["plot"]
+ graph_renderer = plot.handles["glyph_renderer"]
+
+ # Supply widths and heights to the node source
+ node_source = graph_renderer.node_renderer.data_source
+ node_source.data["width"] = element.nodes.data["node_id"].apply(node_width)
+ node_source.data["height"] = [32] * len(element.nodes.data) # 32 px high looks about right
+
+ # Define primary Ellipse glyph
+ graph_renderer.node_renderer.glyph = Ellipse(
+ width="width",
+ height="height",
+ fill_color="white",
+ line_color="gray",
+ )
+
+ # Define hover / inspection Ellipse glyph (prevents reverting to green circles)
+ graph_renderer.node_renderer.hover_glyph = Ellipse(
+ width="width",
+ height="height",
+ fill_color="skyblue",
+ line_color="gray",
+ )
+
+ # Add Arrowheads with matching edge colors
+ for _, row in element.data.iterrows():
+ color = row.get("color", "black")
+ arrow = Arrow(
+ end=NormalHead(fill_color=color, line_color=color, size=8),
+ x_start=row["arrow_starts_x"],
+ y_start=row["arrow_starts_y"],
+ x_end=row["arrow_ends_x"],
+ y_end=row["arrow_ends_y"],
+ line_alpha=0,
+ )
+ fig.add_layout(arrow)
diff --git a/causal_testing/visualisation/testing_dashboard.py b/causal_testing/visualisation/testing_dashboard.py
new file mode 100644
index 00000000..677c1648
--- /dev/null
+++ b/causal_testing/visualisation/testing_dashboard.py
@@ -0,0 +1,224 @@
+"""
+This module implements the Dashboard class to provide a panel dashboard to visualise causal test results.
+"""
+
+import io
+import json
+
+import networkx as nx
+import panel as pn
+import panel_material_ui as pmui
+import param
+
+from causal_testing.causal_testing_framework import CausalTestingFramework, data_readers, read_dataframe
+from causal_testing.specification.causal_dag import CausalDAG
+from causal_testing.testing.causal_test_result import TestOutcome
+from causal_testing.visualisation.visualisation_plotter import VisualisationPlotter
+
+pn.extension(design="material", sizing_mode="stretch_width", notifications=True)
+
+
+class Dashboard(param.Parameterized):
+ """
+ Class to contain the main app.
+ """
+
+ df = param.DataFrame(default=None)
+ ctf = param.ClassSelector(class_=CausalTestingFramework, default=CausalTestingFramework(test_cases=[]))
+ adequacy = param.Boolean(default=False)
+
+ def __init__(self):
+ super().__init__()
+ self.plotter = VisualisationPlotter(self.ctf)
+
+ # DAG
+ self.dag_file_input = pmui.FileInput(accept=".dot,.gv", mime_type="text/vnd.graphviz", label="DAG file")
+ self.dag_file_input.param.watch(self._load_dag_file, "value", onlychanged=True)
+ # Data
+ self.data_file_input = pmui.FileInput(accept=",".join(data_readers), label="Data file")
+ self.data_file_input.param.watch(self._load_data_file, "value", onlychanged=True)
+
+ # Tests
+ self.test_file_input = pmui.FileInput(accept=".json", label="Test file")
+ self.test_file_input.param.watch(self._load_test_file, "value", onlychanged=True)
+
+ # Generate causal tests
+ self.generate_tests = pmui.Button(
+ label="Generate", sizing_mode="fixed", align="end", height=37, width=97, disabled=True
+ )
+ self.generate_tests.param.watch(self._generate_tests, "value", onlychanged=True)
+
+ # Run causal tests
+ self.run_tests = pmui.Button(label="Run Tests", color="primary", disabled=True)
+ self.run_tests.param.watch(self._run_tests, "value", onlychanged=True)
+
+ def _load_dag_file(self, event):
+ """Parses uploaded DOT bytes into a CausalDAG and initialises the CTF with it."""
+ parsed_multigraph = nx.nx_pydot.read_dot(io.StringIO(event.new.decode("utf-8")))
+ dag = CausalDAG()
+ dag.update(nx.DiGraph(parsed_multigraph))
+ self.ctf.dag = dag
+ self.param.trigger("ctf")
+ self.run_tests.disabled = not self.ctf.ready_to_run()
+ self.generate_tests.disabled = False
+
+ def _load_test_file(self, event):
+ self.ctf.test_cases = [self.ctf.create_causal_test(test) for test in json.load(io.BytesIO(event.new))]
+ self.param.trigger("ctf")
+ self.run_tests.disabled = not self.ctf.ready_to_run()
+
+ def _load_data_file(self, event):
+ """Parses uploaded data bytes into a pandas DataFrame and initialises the CTF with it"""
+ self.ctf.df = read_dataframe(file_path=self.data_file_input.filename, content=io.BytesIO(event.new))
+ self.ctf.dag.datatypes = self.ctf.df.dtypes
+ self.run_tests.disabled = not self.ctf.ready_to_run()
+
+ def _generate_tests(self, _):
+ """Generates causal test cases from a DAG."""
+ try:
+ self.ctf.test_cases = self.ctf.dag.generate_causal_tests()
+ self.param.trigger("ctf")
+ self.run_tests.disabled = not self.ctf.ready_to_run()
+ except ValueError as e:
+ pn.state.notifications.error(str(e), duration=0)
+
+ def _run_tests(self, _):
+ self.ctf.run_tests(silent=True, adequacy=self.adequacy)
+ self.param.trigger("ctf")
+
+ def test_suite_stats(self) -> pn.Row:
+ """
+ Key figures about the test suite: Total, Passing, Failing, Inestimable
+ """
+ test_df = self.ctf.test_dataframe()
+ if "result.outcome" not in test_df:
+ test_df["result.outcome"] = None
+
+ num_tests = pn.indicators.Number(
+ name="Test Cases",
+ value=None if test_df.empty else len(test_df),
+ colors=[(0, "black")],
+ styles={"background": "#f8f9fa", "padding": "15px", "border-radius": "8px"},
+ sizing_mode="stretch_width",
+ )
+
+ totals = {outcome: (test_df["result.outcome"] == outcome.name).sum() for outcome in TestOutcome}
+
+ def format_result(outcome: TestOutcome) -> str:
+ if test_df.empty or test_df["result.outcome"].isnull().any():
+ return "-"
+ return f"{{value}} ({(totals[outcome]/len(test_df))*100:.1f}%)"
+
+ return pn.Row(
+ num_tests,
+ pn.indicators.Number(
+ name="Passing tests",
+ value=None if test_df.empty or test_df["result.outcome"].isnull().any() else totals[TestOutcome.PASS],
+ colors=[(1, "red")], # Color red if everything fails
+ default_color="green",
+ format=format_result(TestOutcome.PASS),
+ styles={"background": "#f8f9fa", "padding": "15px", "border-radius": "8px"},
+ sizing_mode="stretch_width",
+ ),
+ pn.indicators.Number(
+ name="Failing tests",
+ value=None if test_df.empty or test_df["result.outcome"].isnull().any() else totals[TestOutcome.FAIL],
+ styles={"background": "#f8f9fa", "padding": "15px", "border-radius": "8px"},
+ colors=[(1, "green")], # Color red if anything fails
+ default_color="red",
+ format=format_result(TestOutcome.FAIL),
+ sizing_mode="stretch_width",
+ ),
+ pn.indicators.Number(
+ name="Inestimable tests",
+ value=(
+ None
+ if test_df.empty or test_df["result.outcome"].isnull().any()
+ else totals[TestOutcome.INESTIMABLE]
+ ),
+ colors=[(1, "green")], # Color green if everything is estimable
+ default_color="orange",
+ format=format_result(TestOutcome.INESTIMABLE),
+ styles={"background": "#f8f9fa", "padding": "15px", "border-radius": "8px"},
+ sizing_mode="stretch_width",
+ ),
+ )
+
+ def sidebar(self) -> pn.Param:
+ """
+ :returns: Parameters to go in the sidebar.
+ """
+ return pn.Column(
+ self.dag_file_input,
+ self.data_file_input,
+ pn.Row(self.test_file_input, self.generate_tests),
+ pn.Param(
+ self.param.adequacy,
+ widgets={
+ "adequacy": pmui.Switch,
+ # "styles": {"transform": "scale(1.5)", "transform-origin": "left center"},
+ },
+ ),
+ self.run_tests,
+ )
+
+ @pn.depends("ctf")
+ def main_panel(self) -> pn.Row:
+ """
+ Main panel for content.
+ """
+ content = pn.Column(self.test_suite_stats())
+
+ if self.ctf.dag is None:
+ content.append(pn.pane.Markdown("Please select the causal DAG."))
+ else:
+ results = pn.Row(
+ self.plotter.interactive_results_dag(
+ width=800,
+ height=450,
+ )
+ )
+
+ if any(test.result for test in self.ctf.test_cases):
+ results.append(
+ self.plotter.test_outcome_adjacency(
+ xrotation=45,
+ width=450,
+ height=450,
+ ),
+ )
+ content.append(results)
+
+ if self.adequacy:
+ content.append(
+ pn.Row(
+ self.plotter.data_adequacy_heatmap(
+ xrotation=45,
+ width=500,
+ height=380,
+ ),
+ self.plotter.dag_adequacy_heatmap(
+ xrotation=45,
+ width=500,
+ height=380,
+ ),
+ ),
+ )
+
+ return content
+
+ def serve(self):
+ """
+ Serve the dashboard.
+ """
+ page_content = pn.template.MaterialTemplate(
+ title="Causal Testing Framework",
+ site="Test Results",
+ sidebar=self.sidebar(),
+ main=[self.main_panel],
+ )
+ pn.serve(page_content, port=5006, show=False)
+
+
+if __name__ == "__main__":
+ Dashboard().serve()
diff --git a/causal_testing/visualisation/visualisation_plotter.py b/causal_testing/visualisation/visualisation_plotter.py
new file mode 100644
index 00000000..531f6800
--- /dev/null
+++ b/causal_testing/visualisation/visualisation_plotter.py
@@ -0,0 +1,330 @@
+"""
+This module implements the visualisation plot generator to generate plots from a CausalTestingFramework instance to help
+visualise the causal test results.
+"""
+
+import holoviews as hv
+import networkx as nx
+import numpy as np
+import pandas as pd
+from bokeh.models import Div, HoverTool
+from bokeh.palettes import RdYlGn
+
+from causal_testing.causal_testing_framework import CausalTestingFramework
+from causal_testing.testing.causal_test_result import TestOutcome
+from causal_testing.visualisation.geometry import edge_spline, node_width, sort_df_by_median_split, style_graph_hook
+
+
+class VisualisationPlotter:
+ """
+ Class to generate plots to visualise CausalTestingFramework test results.
+ """
+
+ def __init__(self, ctf: CausalTestingFramework):
+ self.ctf = ctf
+
+ def results_dag(
+ self,
+ output_file: str = None,
+ view_independences: bool = True,
+ colours: dict[TestOutcome, str] = None,
+ html: bool = False,
+ ) -> nx.DiGraph:
+ """
+ View causal test results as a graph.
+
+ :param output_file: Optional output file to write to (.dot).
+ :param view_independences: Whether to display failed independence tests (defaults to True).
+ :param colours: Optional dictionary of colours to display the test outcomes.
+ By default, pass=green, fail=red, inestimable=orange.
+ :param html: Whether to include html representations of the causal effect. (Defaults to false)
+ """
+ default_colours = {TestOutcome.PASS: "green", TestOutcome.INESTIMABLE: "orange", TestOutcome.FAIL: "red"}
+
+ if colours is not None:
+ colours = default_colours | colours
+ else:
+ colours = default_colours
+
+ result_dag = nx.DiGraph()
+ result_dag.add_nodes_from(self.ctf.dag.nodes)
+ result_dag.add_edges_from(self.ctf.dag.edges)
+
+ for test in self.ctf.test_cases:
+ if test.result:
+ effect_estimate = pd.concat(
+ [
+ test.result.effect_estimate.ci_low,
+ test.result.effect_estimate.effect_estimate,
+ test.result.effect_estimate.ci_high,
+ ],
+ axis=1,
+ )
+ effect_estimate.columns = ["ci_low", "estimate", "ci_high"]
+ if (test.treatment_variable, test.outcome_variable) in result_dag.edges or (
+ view_independences and test.result.outcome != TestOutcome.PASS
+ ):
+ if (test.treatment_variable, test.outcome_variable) not in result_dag.edges:
+ result_dag.add_edge(test.treatment_variable, test.outcome_variable, ignore_cycles=True)
+ result_dag[test.treatment_variable][test.outcome_variable]["style"] = "dashed"
+
+ result_dag[test.treatment_variable][test.outcome_variable]["label"] = test.result.effect_direction()
+ result_dag[test.treatment_variable][test.outcome_variable]["color"] = colours[test.result.outcome]
+ result_dag[test.treatment_variable][test.outcome_variable]["fontcolor"] = colours[
+ test.result.outcome
+ ]
+ if html:
+ effect_estimate = pd.concat(
+ [
+ test.result.effect_estimate.ci_low,
+ test.result.effect_estimate.effect_estimate,
+ test.result.effect_estimate.ci_high,
+ ],
+ axis=1,
+ )
+ effect_estimate.columns = ["ci_low", "estimate", "ci_high"]
+ result_dag[test.treatment_variable][test.outcome_variable]["title"] = effect_estimate.to_html()
+
+ if output_file is not None:
+ nx.drawing.nx_pydot.write_dot(result_dag, output_file)
+
+ return result_dag
+
+ def data_adequacy_heatmap(self, **kwargs) -> hv.HeatMap:
+ """
+ Visualise data adequacy as an adjacency matrix heatmap of the kurtosis.
+ """
+ adequacy = pd.json_normalize(map(lambda t: t.to_dict(), self.ctf.test_cases))
+
+ for col in [
+ "effect_estimate.effect_estimate",
+ "effect_estimate.ci_low",
+ "effect_estimate.ci_high",
+ "adequacy.kurtosis",
+ ]:
+ columns = [c for c in adequacy.columns if c.startswith(f"result.{col}.")]
+ if not columns:
+ return None
+ adequacy[f"result.{col}"] = adequacy[columns].bfill(axis=1).iloc[:, 0]
+ adequacy = adequacy.drop(columns=columns)
+ adequacy = sort_df_by_median_split(adequacy, value_col="result.adequacy.kurtosis")
+
+ # Get data bounds
+ vmin = adequacy["result.adequacy.kurtosis"].min()
+ vmax = adequacy["result.adequacy.kurtosis"].max()
+
+ # Calculate zero position (0.0 to 1.0)
+ zero_ratio = (0 - vmin) / (vmax - vmin)
+
+ # Generate the colour samples from the negative and positive colourmaps
+ num_samples = 1000
+ n_neg = int(num_samples * zero_ratio)
+ n_pos = num_samples - n_neg
+
+ neg_colors = hv.plotting.util.process_cmap("blues_r", provider="bokeh", ncolors=n_neg)
+ pos_colors = hv.plotting.util.process_cmap("YlOrRd", provider="bokeh", ncolors=n_pos)
+ asymmetric_cmap = neg_colors + pos_colors
+
+ # Render
+ return hv.HeatMap(
+ adequacy,
+ kdims=[
+ ("estimator.treatment_variable", "Treatment variable"),
+ ("estimator.outcome_variable", "Outcome variable"),
+ ],
+ vdims=[("result.adequacy.kurtosis", "Kurtosis")],
+ ).opts(
+ cmap=asymmetric_cmap,
+ clim=(vmin, vmax),
+ clipping_colors={"NaN": "grey"}, # Grey out invalid tests
+ colorbar=True,
+ tools=["hover"],
+ xlabel="Treatment variable",
+ ylabel="Outcome variable",
+ clabel="Causal test adequacy",
+ title="Data Adequacy",
+ **kwargs,
+ )
+
+ def dag_adequacy_heatmap(self, **kwargs) -> hv.HeatMap:
+ """
+ Visualise dag adequacy as an adjacency matrix heatmap of the percentage of passing test cases.
+ """
+ adequacy = pd.json_normalize(map(lambda t: t.to_dict(), self.ctf.test_cases))
+ if "result.adequacy.passing" not in adequacy:
+ return None
+
+ # Turn passing test cases into a percentage
+ adequacy["result.adequacy.passing"] = (
+ adequacy["result.adequacy.passing"] / adequacy["result.adequacy.bootstrap_size"]
+ ) * 100
+
+ return hv.HeatMap(
+ sort_df_by_median_split(adequacy, value_col="result.adequacy.passing"),
+ kdims=[
+ ("estimator.treatment_variable", "Treatment variable"),
+ ("estimator.outcome_variable", "Outcome variable"),
+ ],
+ vdims=[("result.adequacy.passing", "Passing (%)")],
+ ).opts(
+ cmap="RdYlGn",
+ clim=(0, 100),
+ clipping_colors={"NaN": "grey"}, # Grey out invalid tests
+ colorbar=True,
+ tools=["hover"],
+ xlabel="Treatment variable",
+ ylabel="Outcome variable",
+ clabel="Percentage passing test cases",
+ title="DAG Adequacy",
+ **kwargs,
+ )
+
+ def test_outcome_adjacency(self, **kwargs) -> hv.HeatMap:
+ """
+ Visualise causal test results as an adjacency matrix.
+ """
+ results = pd.json_normalize(map(lambda t: t.to_dict(), self.ctf.test_cases))
+ results["result.outcome.value"] = results["result.outcome"].apply(lambda x: TestOutcome[x].value)
+
+ green = RdYlGn[11][0]
+ yellow = RdYlGn[11][7]
+ red = RdYlGn[11][10]
+
+ colour_map = {"FAIL": red, "INESTIMABLE": yellow, "PASS": green}
+
+ def add_discrete_legend(plot, _):
+ legend_html = f"""
+
+ ■ Pass
+ ■ Inestimable
+ ■ Fail
+
+ """
+ div = Div(text=legend_html)
+ plot.state.add_layout(div, "above")
+
+ # Apply to your HeatMap
+ return hv.HeatMap(
+ sort_df_by_median_split(results, value_col="result.outcome.value", vdims=["result.outcome"]),
+ kdims=[
+ ("estimator.treatment_variable", "Treatment variable"),
+ ("estimator.outcome_variable", "Outcome variable"),
+ ],
+ vdims=[("result.outcome", "Outcome")],
+ ).opts(
+ cmap=colour_map,
+ clipping_colors={"NaN": "grey"},
+ tools=["hover"],
+ xlabel="Treatment variable",
+ ylabel="Outcome variable",
+ hooks=[add_discrete_legend],
+ title="Test Outcomes",
+ **kwargs,
+ )
+
+ def interactive_results_dag(self, **kwargs) -> hv.Overlay:
+ """
+ Generate an interactive holoview graph of the causal DAG showing failing tests.
+
+ :returns: Inveractive holoviews graph.
+ """
+ results = self.results_dag(html=True)
+
+ # Use DOT to do the layout
+ agraph = nx.nx_agraph.to_agraph(results)
+ agraph.layout(prog="dot")
+
+ node_positions = {}
+ for node in agraph.nodes():
+ x, y = map(float, node.attr["pos"].split(","))
+ node_positions[node.name] = (x, y)
+
+ # Build the edges
+ edges_df = pd.DataFrame([{"source": u, "target": v} | data for u, v, data in results.edges(data=True)])
+
+ edges_df["trimmed_path"] = edges_df[["source", "target"]].apply(
+ lambda row: edge_spline(
+ dot_pos=agraph.get_edge(row["source"], row["target"]).attr["pos"],
+ target_node_centre=node_positions[row["target"]],
+ target_node_width=node_width(row["target"]) / 2,
+ ),
+ axis=1,
+ )
+ edges_df[["arrow_starts_x", "arrow_starts_y"]] = pd.DataFrame(
+ [trimmed_path[-2] for trimmed_path in edges_df["trimmed_path"]], index=edges_df.index
+ )
+ edges_df[["arrow_ends_x", "arrow_ends_y"]] = pd.DataFrame(
+ [trimmed_path[-1] for trimmed_path in edges_df["trimmed_path"]], index=edges_df.index
+ )
+ nodes_df = pd.DataFrame(
+ [(x, y, node_id) for node_id, (x, y) in node_positions.items()], columns=["x", "y", "node_id"]
+ )
+
+ # Build the graph from the nodes and edges
+ graph = hv.Graph(
+ (
+ edges_df,
+ hv.Nodes(
+ nodes_df,
+ kdims=["x", "y", "node_id"],
+ ),
+ hv.EdgePaths(edges_df["trimmed_path"].tolist()),
+ ),
+ kdims=["source", "target"],
+ vdims=[c for c in edges_df.columns if c not in ["source", "target", "trimmed_path"]],
+ ).opts(
+ edge_line_dash="style" if "style" in edges_df else "solid",
+ edge_line_width=1.5,
+ edge_color="color" if "color" in edges_df else "black",
+ edge_hover_line_color="color",
+ hooks=[style_graph_hook],
+ xaxis=None,
+ yaxis=None,
+ tools=[
+ HoverTool(
+ tooltips="""
+
+ Treatment: @source
+ Outcome: @target
+ Causal Effect:
@title{safe}
+
+ """
+ )
+ ],
+ inspection_policy="edges",
+ **kwargs,
+ )
+
+ # Label layers
+ if "label" not in edges_df:
+ edges_df["label"] = ""
+ node_labels = hv.Labels(nodes_df, kdims=["x", "y"], vdims=["node_id"]).opts(
+ text_font_size="9pt",
+ text_color="black",
+ text_align="center",
+ text_baseline="middle",
+ yoffset=0,
+ )
+
+ edge_labels = hv.Labels(
+ pd.concat(
+ [
+ pd.DataFrame(
+ # Stack the x and y elements of the middle index of each trimmed path
+ np.vstack(edges_df["trimmed_path"].apply(lambda path: path[len(path) // 2]).values),
+ columns=["x", "y"],
+ ),
+ edges_df["label"],
+ ],
+ axis=1,
+ ),
+ kdims=["x", "y"],
+ vdims=["label"],
+ ).opts(
+ text_font_size="9pt",
+ text_color="darkblue",
+ text_align="center",
+ text_baseline="middle",
+ )
+
+ return graph * node_labels * edge_labels
diff --git a/docs/source/tutorials/poisson_line_process/poisson_line_process_tutorial.ipynb b/docs/source/tutorials/poisson_line_process/poisson_line_process_tutorial.ipynb
index 6460b21b..d064115f 100644
--- a/docs/source/tutorials/poisson_line_process/poisson_line_process_tutorial.ipynb
+++ b/docs/source/tutorials/poisson_line_process/poisson_line_process_tutorial.ipynb
@@ -371,7 +371,7 @@
" \"height\": estimator.treatment_value,\n",
" \"control\": estimator.control_value,\n",
" \"treatment\": estimator.treatment_value,\n",
- " \"risk_ratio\": causal_test_case.result.effect_estimate.value[0],\n",
+ " \"risk_ratio\": causal_test_case.result.effect_estimate.effect_estimate[0],\n",
" }]"
]
},
@@ -547,7 +547,7 @@
" \"control\": estimator.control_value,\n",
" \"treatment\": estimator.treatment_value,\n",
" \"intensity\": estimator.adjustment_config[\"intensity\"],\n",
- " \"ate\": causal_test_case.result.effect_estimate.value[0],\n",
+ " \"ate\": causal_test_case.result.effect_estimate.effect_estimate[0],\n",
" \"ci_low\": causal_test_case.result.effect_estimate.ci_low[0],\n",
" \"ci_high\": causal_test_case.result.effect_estimate.ci_high[0],\n",
" }]\n"
@@ -763,7 +763,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.15"
+ "version": "3.12.13"
}
},
"nbformat": 4,
diff --git a/docs/source/tutorials/vaccinating_elderly/causal_tests.json b/docs/source/tutorials/vaccinating_elderly/causal_tests.json
index 3c4656c0..30172924 100644
--- a/docs/source/tutorials/vaccinating_elderly/causal_tests.json
+++ b/docs/source/tutorials/vaccinating_elderly/causal_tests.json
@@ -1,109 +1,307 @@
-{
- "tests": [
- {
- "name": "max_doses _||_ cum_vaccinations",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "max_doses",
- "outcome_variable": "cum_vaccinations",
- "expected_effect": {"name": "NoEffect"},
- "estimator_kwargs": {"formula": "cum_vaccinations ~ max_doses"},
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "max_doses _||_ cum_vaccinated",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "max_doses",
- "outcome_variable": "cum_vaccinated",
- "expected_effect": {"name": "NoEffect"},
- "estimator_kwargs": {"formula": "cum_vaccinated ~ max_doses"},
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "max_doses _||_ cum_infections",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "max_doses",
- "outcome_variable": "cum_infections",
- "expected_effect": {"name": "NoEffect"},
- "estimator_kwargs": {"formula": "cum_infections ~ max_doses"},
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "vaccine --> cum_vaccinations",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "vaccine",
- "outcome_variable": "cum_vaccinations",
- "expected_effect": {"name": "SomeEffect"},
- "estimator_kwargs": {"formula": "cum_vaccinations ~ vaccine"},
- "skip": false
- },
- {
- "name": "vaccine --> cum_vaccinated",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "vaccine",
- "outcome_variable": "cum_vaccinated",
- "expected_effect": {"name": "SomeEffect"},
- "estimator_kwargs": {"formula": "cum_vaccinated ~ vaccine"},
- "skip": false
- },
- {
- "name": "vaccine --> cum_infections",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "vaccine",
- "outcome_variable": "cum_infections",
- "expected_effect": {"name": "SomeEffect"},
- "estimator_kwargs": {"formula": "cum_infections ~ vaccine"},
- "skip": false
- },
- {
- "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "cum_vaccinations",
- "outcome_variable": "cum_vaccinated",
- "expected_effect": {"name": "NoEffect"},
- "estimator_kwargs": {"formula": "cum_vaccinated ~ cum_vaccinations + vaccine"},
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "cum_vaccinations _||_ cum_infections | ['vaccine']",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "cum_vaccinations",
- "outcome_variable": "cum_infections",
- "expected_effect": {"name": "NoEffect"},
- "estimator_kwargs": {"formula": "cum_infections ~ cum_vaccinations + vaccine"},
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "cum_vaccinated _||_ cum_infections | ['vaccine']",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "effect": "direct",
- "treatment_variable": "cum_vaccinated",
- "outcome_variable": "cum_infections",
- "expected_effect": {"name": "NoEffect"},
- "estimator_kwargs": {"formula": "cum_infections ~ cum_vaccinated + vaccine"},
- "alpha": 0.05,
- "skip": false
- }
- ]
-}
+[{
+ "name": "max_doses _||_ vaccine",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "max_doses",
+ "outcome_variable": "vaccine",
+ "alpha": 0.05,
+ "formula": "vaccine ~ max_doses"
+ }
+}, {
+ "name": "vaccine _||_ max_doses",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "vaccine",
+ "outcome_variable": "max_doses",
+ "alpha": 0.05,
+ "formula": "max_doses ~ vaccine"
+ }
+}, {
+ "name": "max_doses _||_ cum_vaccinations",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "max_doses",
+ "outcome_variable": "cum_vaccinations",
+ "alpha": 0.05,
+ "formula": "cum_vaccinations ~ max_doses"
+ }
+}, {
+ "name": "cum_vaccinations _||_ max_doses",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_vaccinations",
+ "outcome_variable": "max_doses",
+ "alpha": 0.05,
+ "formula": "max_doses ~ cum_vaccinations"
+ }
+}, {
+ "name": "max_doses _||_ cum_vaccinated",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "max_doses",
+ "outcome_variable": "cum_vaccinated",
+ "alpha": 0.05,
+ "formula": "cum_vaccinated ~ max_doses"
+ }
+}, {
+ "name": "cum_vaccinated _||_ max_doses",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_vaccinated",
+ "outcome_variable": "max_doses",
+ "alpha": 0.05,
+ "formula": "max_doses ~ cum_vaccinated"
+ }
+}, {
+ "name": "max_doses _||_ cum_infections",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "max_doses",
+ "outcome_variable": "cum_infections",
+ "alpha": 0.05,
+ "formula": "cum_infections ~ max_doses"
+ }
+}, {
+ "name": "cum_infections _||_ max_doses",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_infections",
+ "outcome_variable": "max_doses",
+ "alpha": 0.05,
+ "formula": "max_doses ~ cum_infections"
+ }
+}, {
+ "name": "vaccine -> cum_vaccinations",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "SomeEffect",
+ "effect_type": "direct"
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "vaccine",
+ "outcome_variable": "cum_vaccinations",
+ "alpha": 0.05,
+ "formula": "cum_vaccinations ~ vaccine"
+ }
+}, {
+ "name": "vaccine -> cum_vaccinated",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "SomeEffect",
+ "effect_type": "direct"
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "vaccine",
+ "outcome_variable": "cum_vaccinated",
+ "alpha": 0.05,
+ "formula": "cum_vaccinated ~ vaccine"
+ }
+}, {
+ "name": "vaccine -> cum_infections",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "SomeEffect",
+ "effect_type": "direct"
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "vaccine",
+ "outcome_variable": "cum_infections",
+ "alpha": 0.05,
+ "formula": "cum_infections ~ vaccine"
+ }
+}, {
+ "name": "cum_vaccinations _||_ cum_vaccinated",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_vaccinations",
+ "outcome_variable": "cum_vaccinated",
+ "alpha": 0.05,
+ "adjustment_set": ["vaccine"],
+ "formula": "cum_vaccinated ~ cum_vaccinations + vaccine"
+ }
+}, {
+ "name": "cum_vaccinated _||_ cum_vaccinations",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_vaccinated",
+ "outcome_variable": "cum_vaccinations",
+ "alpha": 0.05,
+ "adjustment_set": ["vaccine"],
+ "formula": "cum_vaccinations ~ cum_vaccinated + vaccine"
+ }
+}, {
+ "name": "cum_vaccinations _||_ cum_infections",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_vaccinations",
+ "outcome_variable": "cum_infections",
+ "alpha": 0.05,
+ "adjustment_set": ["vaccine"],
+ "formula": "cum_infections ~ cum_vaccinations + vaccine"
+ }
+}, {
+ "name": "cum_infections _||_ cum_vaccinations",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_infections",
+ "outcome_variable": "cum_vaccinations",
+ "alpha": 0.05,
+ "adjustment_set": ["vaccine"],
+ "formula": "cum_vaccinations ~ cum_infections + vaccine"
+ }
+}, {
+ "name": "cum_vaccinated _||_ cum_infections",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_vaccinated",
+ "outcome_variable": "cum_infections",
+ "alpha": 0.05,
+ "adjustment_set": ["vaccine"],
+ "formula": "cum_infections ~ cum_vaccinated + vaccine"
+ }
+}, {
+ "name": "cum_infections _||_ cum_vaccinated",
+ "skip": false,
+ "effect_measure": "coefficient",
+ "query": null,
+ "expected_causal_effect": {
+ "name": "NoEffect",
+ "effect_type": "direct",
+ "atol": 0,
+ "ctol": 0.0
+ },
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "cum_infections",
+ "outcome_variable": "cum_vaccinated",
+ "alpha": 0.05,
+ "adjustment_set": ["vaccine"],
+ "formula": "cum_vaccinated ~ cum_infections + vaccine"
+ }
+}]
diff --git a/docs/source/tutorials/vaccinating_elderly/test.py b/docs/source/tutorials/vaccinating_elderly/test.py
new file mode 100644
index 00000000..ed634d88
--- /dev/null
+++ b/docs/source/tutorials/vaccinating_elderly/test.py
@@ -0,0 +1,18 @@
+from causal_testing.visualisation.visualisation_plotter import VisualisationPlotter
+from causal_testing.causal_testing_framework import CausalTestingFramework
+
+import holoviews as hv
+import panel as pn
+
+hv.extension("bokeh")
+
+
+DAG_PATH = "dag.dot"
+RESULT_CONFIG = "causal_test_results.json"
+
+framework = CausalTestingFramework()
+framework.setup(dag_path=DAG_PATH, test_cases_path=RESULT_CONFIG)
+
+visualisation_plotter = VisualisationPlotter(framework)
+dag = visualisation_plotter.interactive_results_dag()
+pn.panel(dag).show()
diff --git a/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb b/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb
index b67fe911..7618e15a 100644
--- a/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb
+++ b/docs/source/tutorials/vaccinating_elderly/vaccinating_elderly_tutorial.ipynb
@@ -88,23 +88,25 @@
"\n",
"```json\n",
"{\n",
- " \"tests\": [\n",
- " {\n",
- " \"name\": \"max_doses _||_ cum_vaccinations\",\n",
- " \"estimator\": \"LinearRegressionEstimator\",\n",
- " \"estimate_type\": \"coefficient\",\n",
- " \"effect\": \"direct\",\n",
+ " \"name\": \"max_doses _||_ cum_vaccinations\",\n",
+ " \"skip\": false,\n",
+ " \"effect_measure\": \"coefficient\",\n",
+ " \"query\": null,\n",
+ " \"expected_causal_effect\": {\n",
+ " \"NoEffect\": {\n",
+ " \"effect_type\": \"direct\",\n",
+ " \"atol\": 0,\n",
+ " \"ctol\": 0.0\n",
+ " }\n",
+ " },\n",
+ " \"estimator\": {\n",
+ " \"LinearRegressionEstimator\": {\n",
" \"treatment_variable\": \"max_doses\",\n",
- " \"expected_effect\": {\n",
- " \"cum_vaccinations\": \"NoEffect\"\n",
- " },\n",
- " \"estimator_kwargs\": {\n",
- " \"formula\": \"cum_vaccinations ~ max_doses\",\n",
- " },\n",
+ " \"outcome_variable\": \"cum_vaccinations\",\n",
" \"alpha\": 0.05,\n",
- " \"skip\": false\n",
- " },\n",
- " ]\n",
+ " \"formula\": \"cum_vaccinations ~ max_doses\"\n",
+ " }\n",
+ " }\n",
"}\n",
"```"
]
@@ -129,15 +131,23 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading DAG from dag.dot\n",
- "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - DAG loaded with 5 nodes and 3 edges\n",
- "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading data from 1 source(s)\n",
- "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Initial data shape: (60, 16)\n",
- "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Loading test configurations from causal_tests.json\n",
- "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Running causal tests...\n",
- "100%|████████████████████████████████████████████| 9/9 [00:00<00:00, 271.76it/s]\n",
- "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Saving results to causal_test_results.json\n",
- "2026-07-24 10:44:22 - causal_testing.causal_testing_framework - INFO - Results saved successfully\n"
+ "2026-09-07 15:30:41 - causal_testing.causal_testing_framework - INFO - Loading DAG from dag.dot\n",
+ "2026-09-07 15:30:41 - causal_testing.causal_testing_framework - INFO - DAG loaded with 5 nodes and 3 edges\n",
+ "2026-09-07 15:30:41 - causal_testing.causal_testing_framework - INFO - Loading data from 1 source(s)\n",
+ "2026-09-07 15:30:41 - causal_testing.causal_testing_framework - INFO - Initial data shape: (60, 16)\n",
+ "2026-09-07 15:30:41 - causal_testing.causal_testing_framework - INFO - Loading test configurations from causal_tests.json\n",
+ "2026-09-07 15:30:41 - causal_testing.causal_testing_framework - INFO - Running causal tests...\n",
+ " 0%| | 0/17 [00:00"
+ ""
]
},
"metadata": {},
@@ -190,6 +190,7 @@
"fig, ax = plt.subplots(nrows=len(causal_tests_results), ncols=1, figsize=[10,100])\n",
"\n",
"dag_copy = dag.copy()\n",
+ "dag_copy.ignore_cycles=True\n",
"\n",
"for i, test in enumerate(causal_tests_results):\n",
" treatment_node = test['estimator']['treatment_variable']\n",
@@ -256,7 +257,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.15"
+ "version": "3.12.13"
}
},
"nbformat": 4,
diff --git a/examples/covasim_/doubling_beta/example_beta.py b/examples/covasim_/doubling_beta/example_beta.py
index b9b12116..e70517d2 100644
--- a/examples/covasim_/doubling_beta/example_beta.py
+++ b/examples/covasim_/doubling_beta/example_beta.py
@@ -69,12 +69,12 @@ def doubling_beta_CATE_on_csv(
# Store results for plotting
results_dict["association"] = {
- "ate": causal_test_case.result.effect_estimate.value,
+ "ate": causal_test_case.result.effect_estimate.effect_estimate,
"cis": [causal_test_case.result.effect_estimate.ci_low, causal_test_case.result.effect_estimate.ci_high],
"df": past_execution_df,
}
results_dict["causation"] = {
- "ate": causal_test_case.result.effect_estimate.value,
+ "ate": causal_test_case.result.effect_estimate.effect_estimate,
"cis": [causal_test_case.result.effect_estimate.ci_low, causal_test_case.result.effect_estimate.ci_high],
"df": past_execution_df,
}
@@ -89,7 +89,7 @@ def doubling_beta_CATE_on_csv(
causal_test_case.execute_test(past_execution_df)
results_dict["counterfactual"] = {
- "ate": causal_test_case.result.effect_estimate.value,
+ "ate": causal_test_case.result.effect_estimate.effect_estimate,
"cis": [
causal_test_case.result.effect_estimate.ci_low,
causal_test_case.result.effect_estimate.ci_high,
diff --git a/examples/covasim_/vaccinating_elderly/example_vaccine.py b/examples/covasim_/vaccinating_elderly/example_vaccine.py
index eccdd225..bb508dc2 100644
--- a/examples/covasim_/vaccinating_elderly/example_vaccine.py
+++ b/examples/covasim_/vaccinating_elderly/example_vaccine.py
@@ -55,7 +55,7 @@ def run_test_case(verbose: bool = False):
if verbose:
logging.info("Causation:\n%s", causal_test_case.result)
- results_dict[outcome_variable]["ate"] = causal_test_case.result.effect_estimate.value
+ results_dict[outcome_variable]["ate"] = causal_test_case.result.effect_estimate.effect_estimate
results_dict[outcome_variable]["cis"] = [
causal_test_case.result.effect_estimate.ci_low,
diff --git a/examples/lr91/example_max_conductances.py b/examples/lr91/example_max_conductances.py
index 032f87d2..b121c522 100644
--- a/examples/lr91/example_max_conductances.py
+++ b/examples/lr91/example_max_conductances.py
@@ -101,7 +101,7 @@ def effects_on_APD90(observational_data_path, treatment_var, control_val, treatm
# Run the causal test and print results
causal_test_case.execute_test(pd.read_csv(observational_data_path))
logger.info("%s", causal_test_case.result)
- return causal_test_case.result.effect_estimate.value, (
+ return causal_test_case.result.effect_estimate.effect_estimate, (
causal_test_case.result.effect_estimate.ci_low,
causal_test_case.result.effect_estimate.ci_high,
)
diff --git a/examples/poisson-line-process/causal_tests.json b/examples/poisson-line-process/causal_tests.json
index 3e30194e..b3351b6e 100644
--- a/examples/poisson-line-process/causal_tests.json
+++ b/examples/poisson-line-process/causal_tests.json
@@ -1,226 +1 @@
-{
- "tests": [
- {
- "name": "width --> num_lines_abs",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "width",
- "expected_effect": {
- "num_lines_abs": "SomeEffect"
- },
- "formula": "num_lines_abs ~ width",
- "skip": false
- },
- {
- "name": "width --> num_shapes_abs | ['height', 'num_lines_abs']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "width",
- "expected_effect": {
- "num_shapes_abs": "SomeEffect"
- },
- "formula": "num_shapes_abs ~ width + height + num_lines_abs",
- "skip": false
- },
- {
- "name": "width --> num_lines_unit | ['height', 'num_lines_abs']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "width",
- "expected_effect": {
- "num_lines_unit": "SomeEffect"
- },
- "formula": "num_lines_unit ~ width + height + num_lines_abs",
- "skip": false
- },
- {
- "name": "width --> num_shapes_unit | ['height', 'num_shapes_abs']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "width",
- "expected_effect": {
- "num_shapes_unit": "SomeEffect"
- },
- "formula": "num_shapes_unit ~ width + height + num_shapes_abs",
- "skip": false
- },
- {
- "name": "num_lines_abs --> num_shapes_abs | ['height', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "num_lines_abs",
- "expected_effect": {
- "num_shapes_abs": "SomeEffect"
- },
- "formula": "num_shapes_abs ~ num_lines_abs + height + width",
- "skip": false
- },
- {
- "name": "num_lines_abs --> num_lines_unit | ['height', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "num_lines_abs",
- "expected_effect": {
- "num_lines_unit": "SomeEffect"
- },
- "formula": "num_lines_unit ~ num_lines_abs + height + width",
- "skip": false
- },
- {
- "name": "num_lines_abs _||_ num_shapes_unit | ['height', 'num_shapes_abs', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "num_lines_abs",
- "expected_effect": {
- "num_shapes_unit": "NoEffect"
- },
- "formula": "num_shapes_unit ~ num_lines_abs + height + num_shapes_abs + width",
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "height --> num_lines_abs",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "height",
- "expected_effect": {
- "num_lines_abs": "SomeEffect"
- },
- "formula": "num_lines_abs ~ height",
- "skip": false
- },
- {
- "name": "intensity --> num_lines_abs",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "intensity",
- "expected_effect": {
- "num_lines_abs": "SomeEffect"
- },
- "formula": "num_lines_abs ~ intensity",
- "skip": false
- },
- {
- "name": "num_shapes_abs _||_ num_lines_unit | ['height', 'width', 'num_lines_abs']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "num_shapes_abs",
- "expected_effect": {
- "num_lines_unit": "NoEffect"
- },
- "formula": "num_lines_unit ~ num_shapes_abs + height + width + num_lines_abs",
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "num_shapes_abs --> num_shapes_unit | ['height', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "num_shapes_abs",
- "expected_effect": {
- "num_shapes_unit": "SomeEffect"
- },
- "formula": "num_shapes_unit ~ num_shapes_abs + height + width",
- "skip": false
- },
- {
- "name": "height --> num_shapes_abs | ['num_lines_abs', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "height",
- "expected_effect": {
- "num_shapes_abs": "SomeEffect"
- },
- "formula": "num_shapes_abs ~ height + num_lines_abs + width",
- "skip": false
- },
- {
- "name": "intensity _||_ num_shapes_abs | ['height', 'width', 'num_lines_abs']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "intensity",
- "expected_effect": {
- "num_shapes_abs": "NoEffect"
- },
- "formula": "num_shapes_abs ~ intensity + height + width + num_lines_abs",
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "num_lines_unit _||_ num_shapes_unit | ['height', 'num_shapes_abs', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "num_lines_unit",
- "expected_effect": {
- "num_shapes_unit": "NoEffect"
- },
- "formula": "num_shapes_unit ~ num_lines_unit + height + num_shapes_abs + width",
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "height --> num_lines_unit | ['num_lines_abs', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "height",
- "expected_effect": {
- "num_lines_unit": "SomeEffect"
- },
- "formula": "num_lines_unit ~ height + num_lines_abs + width",
- "skip": false
- },
- {
- "name": "intensity _||_ num_lines_unit | ['height', 'width', 'num_lines_abs']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "intensity",
- "expected_effect": {
- "num_lines_unit": "NoEffect"
- },
- "formula": "num_lines_unit ~ intensity + height + width + num_lines_abs",
- "alpha": 0.05,
- "skip": false
- },
- {
- "name": "height --> num_shapes_unit | ['num_shapes_abs', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "height",
- "expected_effect": {
- "num_shapes_unit": "SomeEffect"
- },
- "formula": "num_shapes_unit ~ height + num_shapes_abs + width",
- "skip": false
- },
- {
- "name": "intensity _||_ num_shapes_unit | ['height', 'num_shapes_abs', 'width']",
- "estimator": "LinearRegressionEstimator",
- "estimate_type": "coefficient",
- "effect": "direct",
- "treatment_variable": "intensity",
- "expected_effect": {
- "num_shapes_unit": "NoEffect"
- },
- "formula": "num_shapes_unit ~ intensity + height + num_shapes_abs + width",
- "alpha": 0.05,
- "skip": false
- }
- ]
-}
\ No newline at end of file
+[{"name": "width -> num_lines_abs", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "width", "outcome_variable": "num_lines_abs", "alpha": 0.05, "formula": "num_lines_abs ~ width"}}, {"name": "width -> num_shapes_abs", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "width", "outcome_variable": "num_shapes_abs", "alpha": 0.05, "adjustment_set": ["height", "num_lines_abs"], "formula": "num_shapes_abs ~ width + height + num_lines_abs"}}, {"name": "width -> num_lines_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "width", "outcome_variable": "num_lines_unit", "alpha": 0.05, "adjustment_set": ["height", "num_lines_abs"], "formula": "num_lines_unit ~ width + height + num_lines_abs"}}, {"name": "width -> num_shapes_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "width", "outcome_variable": "num_shapes_unit", "alpha": 0.05, "adjustment_set": ["height", "num_shapes_abs"], "formula": "num_shapes_unit ~ width + height + num_shapes_abs"}}, {"name": "width _||_ height", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "width", "outcome_variable": "height", "alpha": 0.05, "formula": "height ~ width"}}, {"name": "height _||_ width", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "height", "outcome_variable": "width", "alpha": 0.05, "formula": "width ~ height"}}, {"name": "width _||_ intensity", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "width", "outcome_variable": "intensity", "alpha": 0.05, "formula": "intensity ~ width"}}, {"name": "intensity _||_ width", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "intensity", "outcome_variable": "width", "alpha": 0.05, "formula": "width ~ intensity"}}, {"name": "num_lines_abs -> num_shapes_abs", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "num_lines_abs", "outcome_variable": "num_shapes_abs", "alpha": 0.05, "adjustment_set": ["height", "width"], "formula": "num_shapes_abs ~ num_lines_abs + height + width"}}, {"name": "num_lines_abs -> num_lines_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "num_lines_abs", "outcome_variable": "num_lines_unit", "alpha": 0.05, "adjustment_set": ["height", "width"], "formula": "num_lines_unit ~ num_lines_abs + height + width"}}, {"name": "num_lines_abs _||_ num_shapes_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "num_lines_abs", "outcome_variable": "num_shapes_unit", "alpha": 0.05, "adjustment_set": ["height", "num_shapes_abs", "width"], "formula": "num_shapes_unit ~ num_lines_abs + height + num_shapes_abs + width"}}, {"name": "height -> num_lines_abs", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "height", "outcome_variable": "num_lines_abs", "alpha": 0.05, "formula": "num_lines_abs ~ height"}}, {"name": "intensity -> num_lines_abs", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "intensity", "outcome_variable": "num_lines_abs", "alpha": 0.05, "formula": "num_lines_abs ~ intensity"}}, {"name": "num_shapes_abs _||_ num_lines_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "num_shapes_abs", "outcome_variable": "num_lines_unit", "alpha": 0.05, "adjustment_set": ["height", "num_lines_abs", "width"], "formula": "num_lines_unit ~ num_shapes_abs + height + num_lines_abs + width"}}, {"name": "num_lines_unit _||_ num_shapes_abs", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "num_lines_unit", "outcome_variable": "num_shapes_abs", "alpha": 0.05, "adjustment_set": ["height", "num_lines_abs", "width"], "formula": "num_shapes_abs ~ num_lines_unit + height + num_lines_abs + width"}}, {"name": "num_shapes_abs -> num_shapes_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "num_shapes_abs", "outcome_variable": "num_shapes_unit", "alpha": 0.05, "adjustment_set": ["height", "width"], "formula": "num_shapes_unit ~ num_shapes_abs + height + width"}}, {"name": "height -> num_shapes_abs", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "height", "outcome_variable": "num_shapes_abs", "alpha": 0.05, "adjustment_set": ["num_lines_abs", "width"], "formula": "num_shapes_abs ~ height + num_lines_abs + width"}}, {"name": "intensity _||_ num_shapes_abs", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "intensity", "outcome_variable": "num_shapes_abs", "alpha": 0.05, "adjustment_set": ["height", "num_lines_abs", "width"], "formula": "num_shapes_abs ~ intensity + height + num_lines_abs + width"}}, {"name": "num_lines_unit _||_ num_shapes_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "num_lines_unit", "outcome_variable": "num_shapes_unit", "alpha": 0.05, "adjustment_set": ["height", "num_shapes_abs", "width"], "formula": "num_shapes_unit ~ num_lines_unit + height + num_shapes_abs + width"}}, {"name": "num_shapes_unit _||_ num_lines_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "num_shapes_unit", "outcome_variable": "num_lines_unit", "alpha": 0.05, "adjustment_set": ["height", "num_lines_abs", "width"], "formula": "num_lines_unit ~ num_shapes_unit + height + num_lines_abs + width"}}, {"name": "height -> num_lines_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "height", "outcome_variable": "num_lines_unit", "alpha": 0.05, "adjustment_set": ["num_lines_abs", "width"], "formula": "num_lines_unit ~ height + num_lines_abs + width"}}, {"name": "intensity _||_ num_lines_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "intensity", "outcome_variable": "num_lines_unit", "alpha": 0.05, "adjustment_set": ["height", "num_lines_abs", "width"], "formula": "num_lines_unit ~ intensity + height + num_lines_abs + width"}}, {"name": "height -> num_shapes_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "SomeEffect", "effect_type": "direct"}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "height", "outcome_variable": "num_shapes_unit", "alpha": 0.05, "adjustment_set": ["num_shapes_abs", "width"], "formula": "num_shapes_unit ~ height + num_shapes_abs + width"}}, {"name": "intensity _||_ num_shapes_unit", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "intensity", "outcome_variable": "num_shapes_unit", "alpha": 0.05, "adjustment_set": ["height", "num_shapes_abs", "width"], "formula": "num_shapes_unit ~ intensity + height + num_shapes_abs + width"}}, {"name": "height _||_ intensity", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "height", "outcome_variable": "intensity", "alpha": 0.05, "formula": "intensity ~ height"}}, {"name": "intensity _||_ height", "skip": false, "effect_measure": "coefficient", "query": null, "expected_causal_effect": {"name": "NoEffect", "effect_type": "direct", "atol": 0, "ctol": 0.0}, "estimator": {"name": "LinearRegressionEstimator", "treatment_variable": "intensity", "outcome_variable": "height", "alpha": 0.05, "formula": "height ~ intensity"}}]
\ No newline at end of file
diff --git a/examples/poisson-line-process/example_pure_python.py b/examples/poisson-line-process/example_pure_python.py
index b2d3edae..1a86ebb1 100644
--- a/examples/poisson-line-process/example_pure_python.py
+++ b/examples/poisson-line-process/example_pure_python.py
@@ -43,8 +43,8 @@ def risk_ratio(sample1, sample2):
bootstraps = bootstrap((treatment_results, control_results), risk_ratio, confidence_level=self.alpha)
return EffectEstimate(
- type="risk_ratio",
- value=risk_ratio(treatment_results, control_results),
+ effect_measure="risk_ratio",
+ effect_estimate=risk_ratio(treatment_results, control_results),
ci_low=bootstraps.confidence_interval.low,
ci_high=bootstraps.confidence_interval.high,
)
@@ -101,8 +101,8 @@ def test_poisson_intensity_num_shapes(save=False):
"height": obs_causal_test.estimator.treatment_value,
"control": obs_causal_test.estimator.control_value,
"treatment": obs_causal_test.estimator.treatment_value,
- "smt_risk_ratio": smt_causal_test.result.effect_estimate.value,
- "obs_risk_ratio": obs_causal_test.result.effect_estimate.value[0],
+ "smt_risk_ratio": smt_causal_test.result.effect_estimate.effect_estimate,
+ "obs_risk_ratio": obs_causal_test.result.effect_estimate.effect_estimate[0],
}
for smt_causal_test, _, obs_causal_test in causal_test_cases
]
@@ -138,7 +138,7 @@ def test_poisson_width_num_shapes(save=False):
"control": causal_test.estimator.control_value,
"treatment": causal_test.estimator.treatment_value,
"intensity": causal_test.estimator.adjustment_config["intensity"],
- "ate": causal_test.result.effect_estimate.value[0],
+ "ate": causal_test.result.effect_estimate.effect_estimate[0],
"ci_low": causal_test.result.effect_estimate.ci_low,
"ci_high": causal_test.result.effect_estimate.ci_high,
}
diff --git a/pyproject.toml b/pyproject.toml
index c33ced14..3e3088c9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,7 +21,7 @@ dependencies = [
"pandas>=2.1",
"scikit_learn~=1.4",
"scipy>=1.12.0,<=1.17.1",
- "statsmodels~=0.14",
+ "statsmodels~=0.15",
"tabulate~=0.9",
"pydot>=2.0",
"pygad~=3.3",
@@ -55,6 +55,11 @@ dev = [
"nbformat",
"ipykernel",
]
+visualise = [
+ "holoviews>=1.23",
+ "bokeh>=3.9",
+ "pygraphviz>=2"
+]
[project.urls]
Homepage = "https://sites.google.com/sheffield.ac.uk/citcom/home"
diff --git a/tests/discovery_tests/test_abstract_discovery.py b/tests/discovery_tests/test_abstract_discovery.py
index 1721f400..8448b035 100644
--- a/tests/discovery_tests/test_abstract_discovery.py
+++ b/tests/discovery_tests/test_abstract_discovery.py
@@ -2,20 +2,15 @@
This module tests common causal discovery functionality provided within the abstract_discovery module.
"""
-import os
import unittest
-from tempfile import TemporaryDirectory
import networkx as nx
import pandas as pd
-from numpy import nan
from causal_testing.discovery.abstract_discovery import Discovery, simple_cycle
-from causal_testing.estimation.effect_estimate import EffectEstimate
from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
from causal_testing.specification.causal_dag import CausalDAG
-from causal_testing.testing.causal_test_case import CausalTestCase
-from causal_testing.testing.causal_test_result import CausalTestResult, TestOutcome
+from causal_testing.testing.causal_test_result import TestOutcome
class AbstractDiscovery(Discovery):
@@ -54,46 +49,6 @@ def test_simple_cycle_no_cycles(self):
dag.add_edges_from([("A", "B"), ("B", "C")])
self.assertEqual(simple_cycle(dag), [])
- def test_effect_direction_positive(self):
- causal_test_case = CausalTestCase(
- estimator=LinearRegressionEstimator(treatment_variable="A", outcome_variable="B", adjustment_set=set()),
- effect_measure="ate",
- expected_causal_effect=None,
- )
- causal_test_case.result = CausalTestResult(
- outcome=None,
- effect_estimate=EffectEstimate(
- type="ate", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)
- ),
- )
- self.assertEqual(self.abstract_discovery.effect_direction(causal_test_case), "positive")
-
- def test_effect_direction_negative(self):
- causal_test_case = CausalTestCase(
- estimator=LinearRegressionEstimator(treatment_variable="A", outcome_variable="B", adjustment_set=set()),
- expected_causal_effect=None,
- effect_measure="ate",
- )
- causal_test_case.result = CausalTestResult(
- outcome=None,
- effect_estimate=EffectEstimate(
- type="ate", value=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5)
- ),
- )
- self.assertEqual(self.abstract_discovery.effect_direction(causal_test_case), "negative")
-
- def test_effect_direction_none(self):
- causal_test_case = CausalTestCase(
- estimator=LinearRegressionEstimator(treatment_variable="A", outcome_variable="B", adjustment_set=set()),
- effect_measure="ate",
- expected_causal_effect=None,
- )
- causal_test_case.result = CausalTestResult(
- outcome=None,
- effect_estimate=EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)),
- )
- self.assertEqual(self.abstract_discovery.effect_direction(causal_test_case), None)
-
def test_include_edge_wildcard(self):
abstract_discovery = AbstractDiscovery(
df=pd.DataFrame(columns=["x_1", "x_2", "x_3", "y_1", "y_2", "y_3", "z_1", "z_2"]),
@@ -156,50 +111,6 @@ def test_remove_cycles_multiple_cycles(self):
self.assertTrue(dag.has_edge("A", "B") or dag.has_edge("B", "A"))
self.assertTrue(dag.has_edge("C", "D") or dag.has_edge("D", "C"))
- def test_write_dot(self):
- dag = CausalDAG()
- dag.add_edges_from([("A", "B"), ("C", "D"), ("E", "F")])
- dag.test_results = pd.DataFrame(
- [ # Edges
- {"treatment": "A", "outcome": "B", "effect": "positive", "result": TestOutcome.PASS},
- {"treatment": "C", "outcome": "D", "effect": "positive", "result": TestOutcome.FAIL},
- {"treatment": "E", "outcome": "F", "effect": "None", "result": TestOutcome.INESTIMABLE},
- # Independences
- {"treatment": "A", "outcome": "C", "effect": None, "result": TestOutcome.PASS},
- {"treatment": "A", "outcome": "D", "effect": "negative", "result": TestOutcome.FAIL},
- {"treatment": "A", "outcome": "E", "effect": None, "result": TestOutcome.INESTIMABLE},
- ]
- )
- abstract_discovery = AbstractDiscovery(pd.DataFrame())
- with TemporaryDirectory() as tmp:
- abstract_discovery.write_dot(dag, os.path.join(tmp, "dag.dot"))
- dag2 = CausalDAG(os.path.join(tmp, "dag.dot"))
- self.assertEqual(dag.nodes, dag2.nodes)
-
- def test_write_dot_invalid_edge_outcome(self):
- dag = CausalDAG()
- dag.add_edges_from([("A", "B"), ("C", "D"), ("E", "F")])
- dag.test_results = pd.DataFrame(
- [ # Edges
- {"treatment": "A", "outcome": "B", "effect": None, "result": None},
- ]
- )
- abstract_discovery = AbstractDiscovery(pd.DataFrame())
- with self.assertRaises(ValueError):
- abstract_discovery.write_dot(dag, "dag.dot")
-
- def test_write_dot_invalid_independence_outcome(self):
- dag = CausalDAG()
- dag.add_edges_from([("A", "B"), ("C", "D"), ("E", "F")])
- dag.test_results = pd.DataFrame(
- [ # Edges
- {"treatment": "A", "outcome": "C", "effect": None, "result": None},
- ]
- )
- abstract_discovery = AbstractDiscovery(pd.DataFrame())
- with self.assertRaises(ValueError):
- abstract_discovery.write_dot(dag, "dag.dot")
-
def test_evaluate_tests_invalid_datatype(self):
scarf_df = pd.read_csv("tests/resources/data/scarf_data.csv")
scarf_df["completed"] = pd.to_datetime(["2026-01-01" for _ in range(len(scarf_df))], format="%Y-%m-%d")
@@ -274,7 +185,7 @@ def test_evaluate_tests_inestimable(self):
"outcome": "completed",
},
{
- "result": TestOutcome.INESTIMABLE,
+ "result": TestOutcome.PASS,
"expected_effect": "NoEffect",
"treatment": "color",
"outcome": "completed",
@@ -287,7 +198,6 @@ def test_evaluate_tests_inestimable(self):
},
]
)
- expected_results["effect"] = nan
pd.testing.assert_frame_equal(test_results, expected_results)
def test_evaluate_tests(self):
@@ -363,5 +273,5 @@ def test_evaluate_tests(self):
},
]
)
- expected_results["effect"] = None
+ print(test_results)
pd.testing.assert_frame_equal(test_results, expected_results)
diff --git a/tests/discovery_tests/test_hill_climber_discovery.py b/tests/discovery_tests/test_hill_climber_discovery.py
index 8cd41d13..8fab05dc 100644
--- a/tests/discovery_tests/test_hill_climber_discovery.py
+++ b/tests/discovery_tests/test_hill_climber_discovery.py
@@ -6,7 +6,6 @@
import pandas as pd
-from causal_testing.discovery.abstract_discovery import simple_cycle
from causal_testing.discovery.hill_climber_discovery import HillClimberDiscovery
from causal_testing.specification.causal_dag import CausalDAG
from causal_testing.testing.causal_test_result import TestOutcome
diff --git a/tests/estimation_tests/test_experimental_estimator.py b/tests/estimation_tests/test_experimental_estimator.py
index 394afde7..739ae97f 100644
--- a/tests/estimation_tests/test_experimental_estimator.py
+++ b/tests/estimation_tests/test_experimental_estimator.py
@@ -43,7 +43,7 @@ def test_estimate_ate(self):
repeats=200,
)
effect_estimate = estimator.estimate_ate()
- self.assertEqual(effect_estimate.value["X"], 2)
+ self.assertEqual(effect_estimate.effect_estimate["X"], 2)
self.assertEqual(effect_estimate.ci_low["X"], 2)
self.assertEqual(effect_estimate.ci_high["X"], 2)
@@ -58,6 +58,6 @@ def test_estimate_risk_ratio(self):
repeats=200,
)
effect_estimate = estimator.estimate_risk_ratio()
- self.assertEqual(effect_estimate.value["X"], 2)
+ self.assertEqual(effect_estimate.effect_estimate["X"], 2)
self.assertEqual(effect_estimate.ci_low["X"], 2)
self.assertEqual(effect_estimate.ci_high["X"], 2)
diff --git a/tests/estimation_tests/test_instrumental_variable_estimator.py b/tests/estimation_tests/test_instrumental_variable_estimator.py
index a4e1edf3..9381358e 100644
--- a/tests/estimation_tests/test_instrumental_variable_estimator.py
+++ b/tests/estimation_tests/test_instrumental_variable_estimator.py
@@ -25,12 +25,10 @@ def test_estimate_coefficient(self):
iv_estimator = InstrumentalVariableEstimator(
treatment_variable="X",
outcome_variable="Y",
- treatment_value=None,
- control_value=None,
instrument="Z",
)
effect_estimate = iv_estimator.estimate_coefficient(self.df)
- self.assertEqual(effect_estimate.value[0], 2)
+ self.assertEqual(effect_estimate.effect_estimate[0], 2)
self.assertEqual(effect_estimate.ci_low[0], 2)
self.assertEqual(effect_estimate.ci_high[0], 2)
@@ -38,8 +36,6 @@ def test_to_dict(self):
iv_estimator = InstrumentalVariableEstimator(
treatment_variable="X",
outcome_variable="Y",
- control_value=0,
- treatment_value=1,
instrument="Z",
)
self.assertEqual(
@@ -49,8 +45,6 @@ def test_to_dict(self):
"treatment_variable": "X",
"outcome_variable": "Y",
"alpha": 0.05,
- "control_value": 0,
- "treatment_value": 1,
"instrument": "Z",
"bootstrap_size": 100,
},
diff --git a/tests/estimation_tests/test_ipcw_estimator.py b/tests/estimation_tests/test_ipcw_estimator.py
index 0a3711fb..e65e06f6 100644
--- a/tests/estimation_tests/test_ipcw_estimator.py
+++ b/tests/estimation_tests/test_ipcw_estimator.py
@@ -30,8 +30,8 @@ def test_estimate_hazard_ratio(self):
fit_bltd_switch_formula=self.fit_bl_switch_formula,
eligibility=None,
)
- estimate = estimation_model.estimate_hazard_ratio(self.df)
- self.assertEqual(round(estimate.value["trtrand"], 3), 1.351)
+ effect_estimate = estimation_model.estimate_hazard_ratio(self.df)
+ self.assertEqual(round(effect_estimate.effect_estimate["trtrand"], 3), 1.351)
def test_invalid_treatment_strategies(self):
estimation_model = IPCWEstimator(
diff --git a/tests/estimation_tests/test_linear_regression_estimator.py b/tests/estimation_tests/test_linear_regression_estimator.py
index bfc1fca4..984c9a7d 100644
--- a/tests/estimation_tests/test_linear_regression_estimator.py
+++ b/tests/estimation_tests/test_linear_regression_estimator.py
@@ -169,8 +169,8 @@ def test_program_11_2(self):
# Increasing treatments from 90 to 100 should be the same as 10 times the unit ATE
self.assertTrue(
all(
- round(effect_estimate.value["treatments"], 1) == round(ate_single, 1)
- for ate_single in effect_estimate.value
+ round(effect_estimate.effect_estimate["treatments"], 1) == round(ate_single, 1)
+ for ate_single in effect_estimate.effect_estimate
)
)
@@ -186,8 +186,8 @@ def test_program_11_3(self):
# Increasing treatments from 90 to 100 should be the same as 10 times the unit ATE
self.assertTrue(
all(
- round(effect_estimate.value["treatments"], 3) == round(ate_single, 3)
- for ate_single in effect_estimate.value
+ round(effect_estimate.effect_estimate["treatments"], 3) == round(ate_single, 3)
+ for ate_single in effect_estimate.effect_estimate
)
)
@@ -225,7 +225,7 @@ def test_program_15_1A(self):
)
effect_estimate = linear_regression_estimator.estimate_coefficient(df)
- self.assertEqual(round(effect_estimate.value["qsmk"], 1), 2.6)
+ self.assertEqual(round(effect_estimate.effect_estimate["qsmk"], 1), 2.6)
def test_program_15_no_interaction(self):
"""Test whether our linear regression implementation produces the same results as program 15.1 (p. 163, 184)
@@ -242,7 +242,7 @@ def test_program_15_no_interaction(self):
# for term_to_square in terms_to_square:
effect_estimate = linear_regression_estimator.estimate_coefficient(df)
- self.assertEqual(round(effect_estimate.value.iloc[0], 1), 3.5)
+ self.assertEqual(round(effect_estimate.effect_estimate.iloc[0], 1), 3.5)
self.assertEqual(round(effect_estimate.ci_low.iloc[0], 1), 2.6)
self.assertEqual(round(effect_estimate.ci_high.iloc[0], 1), 4.3)
@@ -260,7 +260,7 @@ def test_program_15_no_interaction_ate(self):
# terms_to_square = ["age", "wt71", "smokeintensity", "smokeyrs"]
# for term_to_square in terms_to_square:
effect_estimate = linear_regression_estimator.estimate_ate(df)
- self.assertEqual(round(effect_estimate.value[0], 1), 3.5)
+ self.assertEqual(round(effect_estimate.effect_estimate[0], 1), 3.5)
self.assertEqual([round(effect_estimate.ci_low[0], 1), round(effect_estimate.ci_high[0], 1)], [2.6, 4.3])
def test_program_15_no_interaction_ate_calculated(self):
@@ -277,7 +277,7 @@ def test_program_15_no_interaction_ate_calculated(self):
)
effect_estimate = linear_regression_estimator.estimate_ate_calculated(df=self.nhefs_df)
- self.assertEqual(round(effect_estimate.value[0], 1), 3.5)
+ self.assertEqual(round(effect_estimate.effect_estimate[0], 1), 3.5)
self.assertEqual([round(effect_estimate.ci_low[0], 1), round(effect_estimate.ci_high[0], 1)], [1.9, 5])
def test_program_11_2_with_robustness_validation(self):
@@ -337,7 +337,7 @@ def test_gp(self):
)
self.assertEqual(linear_regression_estimator.formula, "Y ~ I(1/(X + 1)) - 1")
effect_estimate = linear_regression_estimator.estimate_ate_calculated(df)
- self.assertEqual(round(effect_estimate.value[0], 2), 0.50)
+ self.assertEqual(round(effect_estimate.effect_estimate[0], 2), 0.50)
self.assertEqual(round(effect_estimate.ci_low[0], 2), 0.50)
self.assertEqual(round(effect_estimate.ci_high[0], 2), 0.50)
@@ -360,7 +360,7 @@ def test_gp_power(self):
"Y ~ I(2*X**2) - 1",
)
effect_estimate = linear_regression_estimator.estimate_ate_calculated(df)
- self.assertEqual(round(effect_estimate.value[0], 2), -2.00)
+ self.assertEqual(round(effect_estimate.effect_estimate[0], 2), -2.00)
self.assertEqual(round(effect_estimate.ci_low[0], 2), -2.00)
self.assertEqual(round(effect_estimate.ci_high[0], 2), -2.00)
@@ -388,7 +388,7 @@ def test_X1_effect(self):
formula="Y ~ X1 + X2 + (X1 * X2)",
)
effect_estimate = lr_model.estimate_ate(self.df)
- self.assertAlmostEqual(effect_estimate.value[0], 2.0)
+ self.assertAlmostEqual(effect_estimate.effect_estimate[0], 2.0)
def test_categorical_confidence_intervals(self):
lr_model = LinearRegressionEstimator(
@@ -398,7 +398,9 @@ def test_categorical_confidence_intervals(self):
# The precise values don't really matter. This test is primarily intended to make sure the return type is correct.
self.assertTrue(
- effect_estimate.value.round(2).equals(pd.Series({"color[T.grey]": 0.92, "color[T.orange]": -4.25}))
+ effect_estimate.effect_estimate.round(2).equals(
+ pd.Series({"color[T.grey]": 0.92, "color[T.orange]": -4.25})
+ )
)
self.assertTrue(
effect_estimate.ci_low.round(2).equals(pd.Series({"color[T.grey]": -22.12, "color[T.orange]": -25.58}))
@@ -423,10 +425,10 @@ def test_program_11_3_linear_regression(self):
formula="outcomes ~ cr(treatments, df=3)",
)
- ate_1 = cublic_spline_estimator.estimate_ate_calculated(df).value
+ ate_1 = cublic_spline_estimator.estimate_ate_calculated(df).effect_estimate
cublic_spline_estimator.treatment_value = 2
- ate_2 = cublic_spline_estimator.estimate_ate_calculated(df).value
+ ate_2 = cublic_spline_estimator.estimate_ate_calculated(df).effect_estimate
# Doubling the treatemebnt value should roughly but not exactly double the ATE
self.assertNotEqual(ate_1[0] * 2, ate_2[0])
diff --git a/tests/estimation_tests/test_logistic_regression_estimator.py b/tests/estimation_tests/test_logistic_regression_estimator.py
index eed8719f..656ebe37 100644
--- a/tests/estimation_tests/test_logistic_regression_estimator.py
+++ b/tests/estimation_tests/test_logistic_regression_estimator.py
@@ -23,4 +23,4 @@ def test_odds_ratio(self):
adjustment_set=set(),
)
effect_estimate = logistic_regression_estimator.estimate_unit_odds_ratio(self.scarf_df)
- self.assertEqual(round(effect_estimate.value.iloc[0], 4), 0.8948)
+ self.assertEqual(round(effect_estimate.effect_estimate.iloc[0], 4), 0.8948)
diff --git a/tests/estimation_tests/test_multinomial_regression_estimator.py b/tests/estimation_tests/test_multinomial_regression_estimator.py
index 6b05c63a..20829a13 100644
--- a/tests/estimation_tests/test_multinomial_regression_estimator.py
+++ b/tests/estimation_tests/test_multinomial_regression_estimator.py
@@ -24,7 +24,7 @@ def test_odds_ratio(self):
adjustment_set=set(),
)
effect_estimate = multinomial_regression_estimator.estimate_unit_odds_ratio(self.scarf_df)
- self.assertEqual(round(effect_estimate.value.iloc[0], 4), 0.8948)
+ self.assertEqual(round(effect_estimate.effect_estimate.iloc[0], 4), 0.8948)
def test_odds_ratio_category(self):
multinomial_regression_estimator = MultinomialRegressionEstimator(
@@ -35,7 +35,7 @@ def test_odds_ratio_category(self):
adjustment_set=set(),
)
effect_estimate = multinomial_regression_estimator.estimate_unit_odds_ratio(self.scarf_df)
- self.assertTrue(effect_estimate.value.round(4).equals, pd.Series({"grey": 1.0072, "orange": 0.9668}))
+ self.assertTrue(effect_estimate.effect_estimate.round(4).equals, pd.Series({"grey": 1.0072, "orange": 0.9668}))
def test_odds_ratio_data(self):
multinomial_regression_estimator = MultinomialRegressionEstimator(
@@ -46,4 +46,4 @@ def test_odds_ratio_data(self):
adjustment_set=set(),
)
effect_estimate = multinomial_regression_estimator.estimate_unit_odds_ratio(self.scarf_df)
- self.assertEqual(round(effect_estimate.value.iloc[0], 4), 0.8948)
+ self.assertEqual(round(effect_estimate.effect_estimate.iloc[0], 4), 0.8948)
diff --git a/tests/main_tests/test_ctf.py b/tests/main_tests/test_ctf.py
index 312d3acf..7166720a 100644
--- a/tests/main_tests/test_ctf.py
+++ b/tests/main_tests/test_ctf.py
@@ -56,8 +56,8 @@ def test_create_test_case_invalid_estimator(self):
{
"treatment_variable": "test_input",
"outcome_variable": "test_output",
- "expected_effect": {"name": "NoEffect"},
- "estimator": "InvalidEstimator",
+ "expected_causal_effect": {"name": "NoEffect"},
+ "estimator": {"name": "InvalidEstimator"},
}
)
self.assertEqual(
@@ -75,26 +75,49 @@ def test_create_test_case_no_estimator(self):
{
"treatment_variable": "test_input",
"outcome_variable": "test_output",
- "expected_effect": {"name": "NoEffect"},
+ "expected_causal_effect": {"name": "NoEffect"},
}
)
self.assertEqual(
- "Test configuration must specify an estimator",
+ "Test configuration must specify an `estimator`.",
str(e.exception),
)
+ def test_create_test_case_no_expected_causal_effect(self):
+ framework = CausalTestingFramework()
+ framework.load_dag(self.dag_path)
+ framework.load_data(self.data_paths)
+ test = {
+ "name": "test1",
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "test_input",
+ "outcome_variable": "test_output",
+ "adjustment_set": [],
+ },
+ "effect_measure": "coefficient",
+ }
+ with self.assertRaises(ValueError) as e:
+ framework.create_causal_test(test)
+ self.assertEqual(
+ "Test configuration must specify an `expected_causal_effect`.",
+ str(e.exception),
+ )
+
def test_create_test_case_invalid_effect(self):
framework = CausalTestingFramework()
framework.load_dag(self.dag_path)
framework.load_data(self.data_paths)
test = {
"name": "test1",
- "treatment_variable": "test_input",
- "estimator": "LinearRegressionEstimator",
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "test_input",
+ "outcome_variable": "test_output",
+ "adjustment_set": [],
+ },
"effect_measure": "coefficient",
- "outcome_variable": "test_output",
- "expected_effect": {"name": "InvalidEffect"},
- "estimator_kwargs": {"adjustment_set": []},
+ "expected_causal_effect": {"name": "InvalidEffect"},
}
with self.assertRaises(ValueError) as e:
framework.create_causal_test(test)
@@ -111,12 +134,14 @@ def test_create_test_case_effect_kwargs(self):
framework.load_data(self.data_paths)
test = {
"name": "test1",
- "treatment_variable": "test_input",
- "estimator": "LinearRegressionEstimator",
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "test_input",
+ "outcome_variable": "test_output",
+ "adjustment_set": [],
+ },
"effect_measure": "coefficient",
- "outcome_variable": "test_output",
- "expected_effect": {"name": "ExactValue", "value": 4},
- "estimator_kwargs": {"adjustment_set": []},
+ "expected_causal_effect": {"name": "ExactValue", "value": 4},
}
test_case = framework.create_causal_test(test)
self.assertEqual(test_case.expected_causal_effect.value, 4)
@@ -127,12 +152,14 @@ def test_create_test_case_estimator_kwargs(self):
framework.load_data(self.data_paths)
test = {
"name": "test1",
- "treatment_variable": "test_input",
- "estimator": "InstrumentalVariableEstimator",
+ "estimator": {
+ "name": "InstrumentalVariableEstimator",
+ "treatment_variable": "test_input",
+ "outcome_variable": "test_output",
+ "instrument": "instrumental_variable",
+ },
"effect_measure": "coefficient",
- "outcome_variable": "test_output",
- "expected_effect": {"name": "SomeEffect"},
- "estimator_kwargs": {"instrument": "instrumental_variable"},
+ "expected_causal_effect": {"name": "SomeEffect"},
}
test_case = framework.create_causal_test(test)
self.assertEqual(test_case.estimator.instrument, "instrumental_variable")
@@ -159,7 +186,7 @@ def test_ctf_exception_silent(self):
with open(self.test_cases_path, "r", encoding="utf-8") as f:
test_configs = json.load(f)
- non_skipped_configs = [t for t in test_configs["tests"] if not t.get("skip", False)]
+ non_skipped_configs = [t for t in test_configs if not t.get("skip", False)]
non_skipped_results = [test.result for test in framework.test_cases if not test.skip]
self.assertEqual(len(non_skipped_results), len(non_skipped_configs))
@@ -197,14 +224,14 @@ def test_ctf_evaluate_dag_inestimable(self):
expected = pd.Series(
{
"FAIL": 1,
- "FAIL_ci_high": 2,
+ "FAIL_ci_high": 1,
"FAIL_ci_low": 0,
- "INESTIMABLE": 1,
- "INESTIMABLE_ci_high": 1,
+ "INESTIMABLE": 0,
+ "INESTIMABLE_ci_high": 0,
"INESTIMABLE_ci_low": 0,
- "PASS": 4,
- "PASS_ci_high": 4,
- "PASS_ci_low": 0,
+ "PASS": 5,
+ "PASS_ci_high": 5,
+ "PASS_ci_low": 2,
}
).sort_index()
pd.testing.assert_series_equal(results, expected)
diff --git a/tests/main_tests/test_main.py b/tests/main_tests/test_main.py
index eddb4938..239abe44 100644
--- a/tests/main_tests/test_main.py
+++ b/tests/main_tests/test_main.py
@@ -110,6 +110,47 @@ def test_parse_args_bootstrap_size_explicit_adequacy(self):
executed_tests = [test for test in log if not test.get("skip", False)]
assert all(test["result"].get("bootstrap_size", 50) == 50 for test in executed_tests)
+ def test_parse_args_generate_and_test(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ with patch(
+ "sys.argv",
+ [
+ "causal_testing",
+ "generate",
+ "--dag-path",
+ str(self.dag_path),
+ "--data-paths",
+ str(self.data_paths[0]),
+ "--output",
+ os.path.join(tmp, "tests.json"),
+ ],
+ ):
+ main()
+ self.assertTrue(os.path.exists(os.path.join(tmp, "tests.json")))
+ with patch(
+ "sys.argv",
+ [
+ "causal_testing",
+ "test",
+ "--dag-path",
+ str(self.dag_path),
+ "--data-paths",
+ str(self.data_paths[0]),
+ "--test-config",
+ os.path.join(tmp, "tests.json"),
+ "--output",
+ str(self.output_path.parent / "main.json"),
+ "-A",
+ "-b",
+ "50",
+ ],
+ ):
+ main()
+ with open(self.output_path.parent / "main.json", encoding="utf-8") as f:
+ log = json.load(f)
+ executed_tests = [test for test in log if not test.get("skip", False)]
+ assert all(test["result"].get("bootstrap_size", 50) == 50 for test in executed_tests)
+
def test_parse_args_generation(self):
with tempfile.TemporaryDirectory() as tmp:
with patch(
diff --git a/tests/resources/data/tests.json b/tests/resources/data/tests.json
index 9ee0171f..debc22cd 100644
--- a/tests/resources/data/tests.json
+++ b/tests/resources/data/tests.json
@@ -1,23 +1,31 @@
-{
- "tests": [{
- "name": "test1",
- "treatment_variable": "test_input",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "outcome_variable": "test_output", "expected_effect": {"name": "NoEffect"},
- "skip": false,
- "query": "test_input > 0",
- "estimator_kwargs": {"adjustment_set": []}
-
- },
- {
- "name": "test2",
- "treatment_variable": "test_input",
- "estimator": "LinearRegressionEstimator",
- "effect_measure": "coefficient",
- "outcome_variable": "test_output", "expected_effect": {"name": "NoEffect"},
- "skip": true,
- "query": "test_input <= 5",
- "estimator_kwargs": {"adjustment_set": []}
- }]
-}
+ [{
+ "name": "test1",
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "outcome_variable": "test_output",
+ "treatment_variable": "test_input",
+ "adjustment_set": []
+ },
+ "effect_measure": "coefficient",
+ "expected_causal_effect": {
+ "name": "NoEffect"
+ },
+ "skip": false,
+ "query": "test_input > 0"
+ },
+ {
+ "name": "test2",
+ "estimator": {
+ "name": "LinearRegressionEstimator",
+ "treatment_variable": "test_input",
+ "outcome_variable": "test_output",
+ "adjustment_set": []
+ },
+ "expected_causal_effect": {
+ "name": "NoEffect"
+ },
+ "effect_measure": "coefficient",
+ "skip": true,
+ "query": "test_input <= 5"
+ }
+ ]
diff --git a/tests/testing_tests/test_causal_effect.py b/tests/testing_tests/test_causal_effect.py
index aa910045..275efbbc 100644
--- a/tests/testing_tests/test_causal_effect.py
+++ b/tests/testing_tests/test_causal_effect.py
@@ -13,12 +13,19 @@ class TestCausalEffect(unittest.TestCase):
def setUp(self) -> None:
self.estimator = LinearRegressionEstimator(
- treatment_variable="A", outcome_variable="B", treatment_value=1, control_value=0, adjustment_set=set()
+ treatment_variable="A",
+ outcome_variable="B",
+ treatment_value=1,
+ control_value=0,
+ adjustment_set=set(),
)
def test_effect_estimate_to_dict(self):
effect_estimate = EffectEstimate(
- type="ate", value=pd.Series({"A": 1}), ci_low=pd.Series({"A": 0.1}), ci_high=pd.Series({"A": 1.2})
+ effect_measure="ate",
+ effect_estimate=pd.Series({"A": 1}),
+ ci_low=pd.Series({"A": 0.1}),
+ ci_high=pd.Series({"A": 1.2}),
)
self.assertEqual(
effect_estimate.to_dict(),
@@ -26,44 +33,50 @@ def test_effect_estimate_to_dict(self):
)
def test_effect_estimate_to_dict_no_ci(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series({"A": 1}))
+ effect_estimate = EffectEstimate(effect_measure="ate", effect_estimate=pd.Series({"A": 1}))
self.assertEqual(
effect_estimate.to_dict(),
{"effect_measure": "ate", "effect_estimate": {"A": 1}},
)
def test_Positive_ate_pass(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6))
+ effect_estimate = EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)
+ )
self.assertTrue(Positive().apply(effect_estimate))
def test_Positive_risk_ratio_pass(self):
effect_estimate = EffectEstimate(
- type="risk_ratio", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)
+ effect_measure="risk_ratio", effect_estimate=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)
)
self.assertTrue(Positive().apply(effect_estimate))
def test_Positive_fail(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1))
+ effect_estimate = EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)
+ )
self.assertFalse(Positive().apply(effect_estimate))
def test_Negative_ate_pass(self):
effect_estimate = EffectEstimate(
- type="ate", value=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5)
+ effect_measure="ate", effect_estimate=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5)
)
self.assertTrue(Negative().apply(effect_estimate))
def test_Negative_risk_ratio_pass(self):
effect_estimate = EffectEstimate(
- type="risk_ratio", value=pd.Series(0.2), ci_low=pd.Series(0.1), ci_high=pd.Series(0.5)
+ effect_measure="risk_ratio", effect_estimate=pd.Series(0.2), ci_low=pd.Series(0.1), ci_high=pd.Series(0.5)
)
self.assertTrue(Negative().apply(effect_estimate))
def test_Negative_fail(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1))
+ effect_estimate = EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)
+ )
self.assertFalse(Negative().apply(effect_estimate))
def test_exactValue_pass(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series(5.05))
+ effect_estimate = EffectEstimate(effect_measure="ate", effect_estimate=pd.Series(5.05))
self.assertTrue(ExactValue(value=5, atol=0.1).apply(effect_estimate))
def test_exactValue_to_dict(self):
@@ -73,29 +86,33 @@ def test_exactValue_to_dict(self):
)
def test_exactValue_categorical_pass(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series({"color[T.red]": 5.05, "color[T.blue]": 4.03}))
+ effect_estimate = EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series({"color[T.red]": 5.05, "color[T.blue]": 4.03})
+ )
self.assertTrue(
ExactValue(value=pd.Series({"color[T.red]": 5, "color[T.blue]": 4}), atol=0.1).apply(effect_estimate)
)
def test_exactValue_pass_ci(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series(5.05), ci_low=pd.Series(4), ci_high=pd.Series(6))
+ effect_estimate = EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(5.05), ci_low=pd.Series(4), ci_high=pd.Series(6)
+ )
self.assertTrue(ExactValue(value=5, atol=0.1).apply(effect_estimate))
def test_exactValue_ci_pass_ci(self):
effect_estimate = EffectEstimate(
- type="ate", value=pd.Series(5.05), ci_low=pd.Series(4.1), ci_high=pd.Series(5.9)
+ effect_measure="ate", effect_estimate=pd.Series(5.05), ci_low=pd.Series(4.1), ci_high=pd.Series(5.9)
)
self.assertTrue(ExactValue(value=5, atol=0.05, ci_low=4, ci_high=6).apply(effect_estimate))
def test_exactValue_ci_fail_ci(self):
effect_estimate = EffectEstimate(
- type="ate", value=pd.Series(5.05), ci_low=pd.Series(4.1), ci_high=pd.Series(5.9)
+ effect_measure="ate", effect_estimate=pd.Series(5.05), ci_low=pd.Series(4.1), ci_high=pd.Series(5.9)
)
self.assertFalse(ExactValue(value=5, atol=0.04, ci_low=4, ci_high=6).apply(effect_estimate))
def test_exactValue_fail(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series(0))
+ effect_estimate = EffectEstimate(effect_measure="ate", effect_estimate=pd.Series(0))
self.assertFalse(ExactValue(value=5, atol=0.1).apply(effect_estimate))
def test_invalid_atol(self):
@@ -120,7 +137,7 @@ def test_invalid_ci_atol(self):
def test_invalid(self):
effect_estimate = EffectEstimate(
- type="invalid", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7)
+ effect_measure="invalid", effect_estimate=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7)
)
with self.assertRaises(ValueError):
SomeEffect().apply(effect_estimate)
@@ -133,27 +150,29 @@ def test_invalid(self):
def test_someEffect_pass_coefficient(self):
effect_estimate = EffectEstimate(
- type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7)
+ effect_measure="coefficient", effect_estimate=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7)
)
self.assertTrue(SomeEffect().apply(effect_estimate))
self.assertFalse(NoEffect().apply(effect_estimate))
def test_someEffect_pass_ate(self):
effect_estimate = EffectEstimate(
- type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7)
+ effect_measure="coefficient", effect_estimate=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7)
)
self.assertTrue(SomeEffect().apply(effect_estimate))
self.assertFalse(NoEffect().apply(effect_estimate))
def test_someEffect_pass_rr(self):
effect_estimate = EffectEstimate(
- type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7)
+ effect_measure="coefficient", effect_estimate=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7)
)
self.assertTrue(SomeEffect().apply(effect_estimate))
self.assertFalse(NoEffect().apply(effect_estimate))
def test_someEffect_fail(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-0.1), ci_high=pd.Series(0.2))
+ effect_estimate = EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(0), ci_low=pd.Series(-0.1), ci_high=pd.Series(0.2)
+ )
self.assertFalse(SomeEffect().apply(effect_estimate))
self.assertTrue(NoEffect().apply(effect_estimate))
@@ -178,7 +197,7 @@ def test_negative_risk_ratio_e_value_using_ci(self):
self.assertEqual(round(e_value, 4), 1.4625)
def test_multiple_value_exception_caught(self):
- effect_estimate = EffectEstimate(type="ate", value=pd.Series([0, 1]))
+ effect_estimate = EffectEstimate(effect_measure="ate", effect_estimate=pd.Series([0, 1]))
with self.assertRaises(ValueError):
Positive().apply(effect_estimate)
with self.assertRaises(ValueError):
diff --git a/tests/testing_tests/test_causal_test_adequacy.py b/tests/testing_tests/test_causal_test_adequacy.py
index 2aec4a40..218e8942 100644
--- a/tests/testing_tests/test_causal_test_adequacy.py
+++ b/tests/testing_tests/test_causal_test_adequacy.py
@@ -73,6 +73,7 @@ def test_data_adequacy_categorical_inestimable(self):
),
)
adequacy_metric = causal_test_case.measure_adequacy(df.loc[df["color"] == "grey"])
+ print(adequacy_metric.kurtosis)
self.assertEqual(adequacy_metric.kurtosis, None, f"Expected passing None not {adequacy_metric.kurtosis}")
self.assertEqual(adequacy_metric.passing, 0, f"Expected passing 0 not {adequacy_metric.passing}")
@@ -210,7 +211,7 @@ def test_to_dict(self):
# Use json_normalize to avoid rounding errors
pd.testing.assert_frame_equal(
pd.json_normalize(expected_dict).round(2),
- pd.json_normalize(adequacy_metric.to_dict(include_results=True)).round(2),
+ pd.json_normalize(adequacy_metric.to_dict(include_adequacy_results=True)).round(2),
)
def test_dag_adequacy_dependent(self):
diff --git a/tests/testing_tests/test_causal_test_case.py b/tests/testing_tests/test_causal_test_case.py
index 7908001e..465c0f36 100644
--- a/tests/testing_tests/test_causal_test_case.py
+++ b/tests/testing_tests/test_causal_test_case.py
@@ -1,6 +1,3 @@
-import os
-import shutil
-import tempfile
import unittest
import numpy as np
@@ -72,13 +69,13 @@ def test_execute_test_observational_linear_regression_estimator(self):
effect_measure="ate",
)
effect_estimate = causal_test_case.estimate_effect(self.df)
- pd.testing.assert_series_equal(effect_estimate.value, pd.Series(4.0), atol=1e-10)
+ pd.testing.assert_series_equal(effect_estimate.effect_estimate, pd.Series(4.0), atol=1e-10)
def test_execute_test_observational_linear_regression_estimator_direct_effect(self):
"""Check that executing the causal test case returns the correct results for dummy data using a linear
regression estimator."""
effect_estimate = self.causal_test_case.estimate_effect(self.df)
- pd.testing.assert_series_equal(effect_estimate.value, pd.Series(4.0), atol=1e-10)
+ pd.testing.assert_series_equal(effect_estimate.effect_estimate, pd.Series(4.0), atol=1e-10)
def test_execute_test_observational_linear_regression_estimator_coefficient(self):
"""Check that executing the causal test case returns the correct results for dummy data using a linear
@@ -95,7 +92,7 @@ def test_execute_test_observational_linear_regression_estimator_coefficient(self
effect_measure="coefficient",
)
effect_estimate = causal_test_case.estimate_effect(self.df)
- pd.testing.assert_series_equal(effect_estimate.value, pd.Series({"D": 0.0}), atol=1e-1)
+ pd.testing.assert_series_equal(effect_estimate.effect_estimate, pd.Series({"D": 0.0}), atol=1e-1)
def test_execute_test_observational_linear_regression_estimator_risk_ratio(self):
"""Check that executing the causal test case returns the correct results for dummy data using a linear
@@ -112,7 +109,7 @@ def test_execute_test_observational_linear_regression_estimator_risk_ratio(self)
effect_measure="risk_ratio",
)
effect_estimate = causal_test_case.estimate_effect(self.df)
- pd.testing.assert_series_equal(effect_estimate.value, pd.Series(0.0), atol=1)
+ pd.testing.assert_series_equal(effect_estimate.effect_estimate, pd.Series(0.0), atol=1)
def test_invalid_effect_measure(self):
"""Check that executing the causal test case returns the correct results for dummy data using a linear
@@ -147,7 +144,7 @@ def test_execute_test_observational_linear_regression_estimator_squared_term(sel
effect_measure="ate",
)
effect_estimate = causal_test_case.estimate_effect(self.df)
- pd.testing.assert_series_equal(effect_estimate.value, pd.Series(4.0), atol=1)
+ pd.testing.assert_series_equal(effect_estimate.effect_estimate, pd.Series(4.0), atol=1)
def test_estimate_params_with_formula(self):
"""Ensure estimate params is handled correctly when a formula is passed into the estimator object"""
@@ -166,7 +163,7 @@ def test_estimate_params_with_formula(self):
)
self.assertEqual(
round(
- causal_test_case.estimate_effect(self.df).value[0],
+ causal_test_case.estimate_effect(self.df).effect_estimate[0],
3,
),
1.444,
@@ -191,7 +188,7 @@ def test_to_dict(self):
"skip": False,
"effect_measure": "coefficient",
"query": None,
- "expected_effect": {"name": "ExactValue", "effect_type": "direct", "value": 4, "atol": 0},
+ "expected_causal_effect": {"name": "ExactValue", "effect_type": "direct", "value": 4, "atol": 0},
"estimator": {
"name": "LinearRegressionEstimator",
"treatment_variable": "A",
@@ -203,14 +200,19 @@ def test_to_dict(self):
"result": {
"outcome": "PASS",
"passed": True,
- "effect_measure": "coefficient",
- "effect_estimate": {"A": 4.0},
- "ci_low": {"A": 4.0},
- "ci_high": {"A": 4.0},
+ "effect_estimate": {
+ "effect_measure": "coefficient",
+ "effect_estimate": {"A": 4.0},
+ "ci_low": {"A": 4.0},
+ "ci_high": {"A": 4.0},
+ },
"adequacy": {"kurtosis": {"A": 0.0}, "passing": 100, "successful": 100, "bootstrap_size": 100},
},
}
+ print([k for k in expected if k not in test_case_dict])
+ print([k for k in test_case_dict if k not in expected])
+
# Use json_normalize to avoid rounding errors
pd.testing.assert_frame_equal(
pd.json_normalize(expected).round(2),
diff --git a/tests/testing_tests/test_causal_test_result.py b/tests/testing_tests/test_causal_test_result.py
new file mode 100644
index 00000000..6d0050a7
--- /dev/null
+++ b/tests/testing_tests/test_causal_test_result.py
@@ -0,0 +1,52 @@
+"""
+Test the CausalTestResult class.
+"""
+
+import unittest
+
+import pandas as pd
+
+from causal_testing.estimation.effect_estimate import EffectEstimate
+from causal_testing.testing.causal_test_result import CausalTestResult
+
+
+class TestCausalTestCase(unittest.TestCase):
+
+ def test_effect_direction_positive(self):
+ result = CausalTestResult(
+ outcome=None,
+ effect_estimate=EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)
+ ),
+ )
+ self.assertEqual(result.effect_direction(), "positive")
+
+ def test_effect_direction_negative(self):
+ result = CausalTestResult(
+ outcome=None,
+ effect_estimate=EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5)
+ ),
+ )
+ self.assertEqual(result.effect_direction(), "negative")
+
+ def test_effect_direction_none(self):
+ result = CausalTestResult(
+ outcome=None,
+ effect_estimate=EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)
+ ),
+ )
+ self.assertEqual(result.effect_direction(), "no effect")
+
+ def test_effect_direction_categorical(self):
+ result = CausalTestResult(
+ outcome=None,
+ effect_estimate=EffectEstimate(
+ effect_measure="ate",
+ effect_estimate=pd.Series({"color[T.RED]": -5, "color[T.BLUE]": -4}),
+ ci_low=pd.Series({"color[T.RED]": -4, "color[T.BLUE]": -1}),
+ ci_high=pd.Series({"color[T.RED]": 5, "color[T.BLUE]": 4}),
+ ),
+ )
+ self.assertEqual(result.effect_direction(), "categorical")
diff --git a/tests/visualisation_tests/test_causal_test_result_visualiser.py b/tests/visualisation_tests/test_causal_test_result_visualiser.py
new file mode 100644
index 00000000..098badda
--- /dev/null
+++ b/tests/visualisation_tests/test_causal_test_result_visualiser.py
@@ -0,0 +1,47 @@
+import os
+import unittest
+from itertools import cycle
+from tempfile import TemporaryDirectory
+
+import pandas as pd
+
+from causal_testing.causal_testing_framework import CausalTestingFramework
+from causal_testing.estimation.effect_estimate import EffectEstimate
+from causal_testing.specification.causal_dag import CausalDAG
+from causal_testing.testing.causal_test_result import CausalTestResult, TestOutcome
+from causal_testing.visualisation.visualisation_plotter import VisualisationPlotter
+
+
+class TestVisualiser(unittest.TestCase):
+ def test_results_dag(self):
+ dag = CausalDAG()
+ dag.add_edges_from([("A", "B"), ("C", "D"), ("E", "F")])
+ dag.datatypes = {node: float for node in dag.nodes}
+ test_cases = dag.generate_causal_tests()
+
+ test_result_cycle = cycle([TestOutcome.PASS, TestOutcome.FAIL, TestOutcome.INESTIMABLE])
+ effect_estimate_cycle = cycle(
+ [
+ EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(5), ci_low=pd.Series(4), ci_high=pd.Series(6)
+ ), # Positive
+ EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(5), ci_low=pd.Series(-4), ci_high=pd.Series(6)
+ ), # No effect
+ EffectEstimate(
+ effect_measure="ate", effect_estimate=pd.Series(-5), ci_low=pd.Series(-6), ci_high=pd.Series(-4)
+ ), # Negative
+ ]
+ )
+ for test in test_cases:
+ test.result = CausalTestResult(
+ effect_estimate=next(effect_estimate_cycle),
+ outcome=next(test_result_cycle),
+ )
+ ctf = CausalTestingFramework(dag=dag, test_cases=test_cases)
+ vp = VisualisationPlotter(ctf)
+
+ with TemporaryDirectory() as tmp:
+ vp.results_dag(output_file=os.path.join(tmp, "dag.dot"))
+ dag2 = CausalDAG(os.path.join(tmp, "dag.dot"), ignore_cycles=True)
+ self.assertEqual(dag.nodes, dag2.nodes)