diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 7d692c1..993ebf4 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -12,6 +12,9 @@ concurrency: group: "${{ github.ref }}-${{ github.head_ref }}-${{ github.workflow }}" cancel-in-progress: false +permissions: + contents: read + jobs: build: name: Build package diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f548f62..54adc92 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,6 +14,8 @@ concurrency: jobs: docs: runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7a12d26..7dd1395 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [ main, develop ] +permissions: + contents: read + jobs: lint: runs-on: ubuntu-latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8b09f6e..69fdc44 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,9 @@ on: schedule: - cron: '0 0 * * *' +permissions: + contents: read + jobs: build: diff --git a/Plans/person3-ml-workplan-2026-07-13.md b/Plans/person3-ml-workplan-2026-07-13.md new file mode 100644 index 0000000..7a5b887 --- /dev/null +++ b/Plans/person3-ml-workplan-2026-07-13.md @@ -0,0 +1,173 @@ +# Person 3 Work Plan (ML Training & Evaluation) + +**Owner:** Chenou +**Date:** 2026-07-13 +**Branch:** person3-ml-training-eval +**Target milestone:** Phase 3 complete by 2026-07-27 + +--- + +## Scope + +Build the end-to-end ML pipeline for ConfoState: + +1. Dataset assembly and split logic +2. Baseline model training +3. Evaluation + report generation +4. Model artifact management / registry + +Primary deliverables: +- confostate/data/datasets.py +- confostate/models/baseline.py +- confostate/models/train.py +- confostate/models/evaluate.py +- confostate/models/registry.py +- tests for datasets/model training/evaluation +- trained artifacts in data/models/ +- evaluation report in docs/reports/ + +--- + +## Dependencies and Risk Controls + +### Needed from Person 1 and 2 +- Stable annotations schema in data/annotations/leu_t_transporters.csv +- Feature vectors for training rows (or at least a subset) + +### Risk if features are delayed +- Use a synthetic feature table to unblock model/training/evaluation development. +- Keep synthetic schema aligned with expected real feature names. +- Add a quick integration check that can switch from synthetic to real features by changing only file paths. + +### Definition of done for integration +- Train command runs successfully from raw tabular features + labels. +- Evaluation command produces metrics and confusion matrix artifact. +- Registry points to a reproducible model artifact and metadata. + +--- + +## Week-by-Week Plan + +## Week 1 (2026-07-13 to 2026-07-19): Pipeline Skeleton + Baselines + +### Day 1: Data contract and dataset loader +- Finalize expected columns: + - keys: pdb_id, family, conformation + - features: numeric columns only +- Implement confostate/data/datasets.py: + - load_dataset(annotations_csv, features_csv) + - validate label and key integrity + - make_split(strategy="stratified", test_size=0.2, random_state=...) + +### Day 2: Baseline model module +- Implement confostate/models/baseline.py: + - logistic regression baseline + - random forest baseline + - SVM baseline +- Add uniform interface: + - get_model(name, random_state, class_weight) + +### Day 3: Training pipeline +- Implement confostate/models/train.py: + - fit single model + - cross-validation score summary + - save artifact + metadata +- Save outputs under data/models/{family}/{model_name}/ + +### Day 4: Evaluation module +- Implement confostate/models/evaluate.py: + - confusion matrix + - per-class precision/recall/F1 + - macro and weighted scores + - optional ROC-AUC (when applicable) +- Emit markdown report to docs/reports/ + +### Day 5: Smoke tests + first run +- Add tests for: + - dataset split stability + - model train/predict shape and class consistency + - evaluation output keys and file generation +- Run first end-to-end experiment with available data or synthetic fallback + +## Week 2 (2026-07-20 to 2026-07-27): Hardening + Reporting + Handoff + +### Day 6-7: Hyperparameter tuning and comparison +- Add small grid/random search for each baseline model +- Compare by macro-F1 and balanced accuracy +- Select default baseline for each family + +### Day 8: Registry and reproducibility +- Implement confostate/models/registry.py: + - register_model(family, model_name, artifact_path, metrics, data_version) + - load_registered_model(family) +- Include metadata fields: + - timestamp, git commit hash, feature schema hash, random_state + +### Day 9: Final report package +- Produce docs/reports/phase3-baseline-report.md +- Include: + - train/test split method + - model comparison table + - per-state metrics + - known limitations and next steps + +### Day 10: Handoff to Person 4 and Person 5 +- Share: + - top model artifacts + - feature importance-compatible model outputs + - stable prediction API contract for explainability and CLI + +--- + +## Suggested File Interfaces + +### confostate/data/datasets.py +- load_dataset(annotations_csv: str, features_csv: str, family: str | None = None) +- train_test_split_dataset(df, label_col="conformation", test_size=0.2, random_state=42) + +### confostate/models/baseline.py +- get_baseline_models(random_state=42) -> dict[str, estimator] +- train_model(estimator, X_train, y_train) + +### confostate/models/train.py +- run_training(config: dict) -> dict +- save_model_artifact(model, out_dir, metadata) + +### confostate/models/evaluate.py +- evaluate_model(model, X_test, y_test, labels=None) -> dict +- write_evaluation_report(metrics: dict, output_path: str) + +### confostate/models/registry.py +- register_model(...) +- get_registered_model(family: str) + +--- + +## Daily Execution Checklist + +- Confirm branch and pull latest main changes (if needed) +- Implement one scoped unit of functionality +- Add or update tests +- Run local test subset +- Commit with concise message +- Post async update in team channel with blockers/dependencies + +--- + +## Communication Cadence + +- Tuesday standup: report progress, current blocker, next 48h plan +- Async updates: at least every 2 working days in #confostate-dev +- Dependency syncs: + - with Person 2 on feature table schema + - with Person 4 on importance/explanation-ready outputs + +--- + +## Exit Criteria (Phase 3 Complete) + +- End-to-end training command works on at least one family +- >=2 baseline models compared and documented +- Evaluation report generated with per-state metrics +- Model registry points to reproducible artifacts +- Person 4 receives model + outputs needed for explainability diff --git a/Plans/person3-progress-memo-2026-07-13.md b/Plans/person3-progress-memo-2026-07-13.md new file mode 100644 index 0000000..2ad873b --- /dev/null +++ b/Plans/person3-progress-memo-2026-07-13.md @@ -0,0 +1,51 @@ +# Person 3 Progress Memo + +**Date:** 2026-07-13 +**Owner:** Chenou +**Branch:** person3-ml-training-eval + +## Summary + +Implemented an initial, runnable scaffold for Person 3 (ML Training & Evaluation) so the team can start end-to-end model development before all upstream dependencies are finalized. + +## Completed Work + +1. Added detailed Person 3 execution plan and linked it from the 6-person team plan. +2. Added dataset utilities for annotation/features merge, split generation, and X/y extraction. +3. Added baseline model, training, evaluation, and registry modules under `confostate/models`. +4. Added outline scripts for each Person 3 workstream task in `scripts/`. +5. Updated package exports and dependencies to include model subpackages and `scikit-learn`. +6. Documented script usage in `docs/USAGE.md`. + +## New/Updated Paths + +- Plans/person3-ml-workplan-2026-07-13.md +- Plans/workplan-6person.md +- confostate/data/datasets.py +- confostate/models/__init__.py +- confostate/models/baseline.py +- confostate/models/train.py +- confostate/models/evaluate.py +- confostate/models/registry.py +- confostate/__init__.py +- confostate/data/__init__.py +- scripts/p3_dataset_loader.py +- scripts/p3_baseline_models.py +- scripts/p3_training_pipeline.py +- scripts/p3_evaluate_reporting.py +- scripts/p3_model_registry.py +- docs/USAGE.md +- pyproject.toml + +## Notes + +- The scripts are intentionally outline-level and designed for iterative refinement. +- Current workflow supports a synthetic feature table fallback if upstream feature extraction is delayed. +- Preferred test environment for ConfoState runs: + `/nfs/homes5/Projects/SLC26/chenou/openff/UGM2025/workshops/OpenFF/micromamba_root/envs/ConfoState` + +## Immediate Next Steps + +1. Add a minimal synthetic features CSV fixture for smoke testing. +2. Run one full train/eval/registry cycle and capture report artifacts. +3. Add pytest smoke tests for dataset merge and training script execution. diff --git a/Plans/workplan-6person.md b/Plans/workplan-6person.md index 7d98acf..f2cd98f 100644 --- a/Plans/workplan-6person.md +++ b/Plans/workplan-6person.md @@ -16,6 +16,7 @@ ConfoState development split into 6 parallel work streams, each led by one team - 2026-06-15 initial draft (AI generated) - 2026-06-29 annotated in group meeting +- 2026-07-13 Person 3 execution plan added in Plans/person3-ml-workplan-2026-07-13.md ## General development notes @@ -176,6 +177,8 @@ ConfoState development split into 6 parallel work streams, each led by one team **Assignee**: Chenou +Detailed execution plan: Plans/person3-ml-workplan-2026-07-13.md + ### Tasks 1. **Build dataset loader** diff --git a/confostate/__init__.py b/confostate/__init__.py index dc646bb..ea46ba0 100644 --- a/confostate/__init__.py +++ b/confostate/__init__.py @@ -4,5 +4,22 @@ from confostate.data.loader import load_annotations from confostate.features import extract_features +from confostate.models import ( + evaluate_model, + get_baseline_models, + get_registered_model, + register_model, + run_training, + write_evaluation_report, +) -__all__ = ["load_annotations", "extract_features"] +__all__ = [ + "extract_features", + "evaluate_model", + "get_baseline_models", + "get_registered_model", + "load_annotations", + "register_model", + "run_training", + "write_evaluation_report", +] diff --git a/confostate/data/__init__.py b/confostate/data/__init__.py index d0894e7..64c369f 100644 --- a/confostate/data/__init__.py +++ b/confostate/data/__init__.py @@ -1 +1,16 @@ """Data loading utilities for ConfoState.""" + +from confostate.data.datasets import ( + build_xy, + load_dataset, + train_test_split_dataset, +) +from confostate.data.loader import load_annotations, load_from_input_dir + +__all__ = [ + "build_xy", + "load_annotations", + "load_dataset", + "load_from_input_dir", + "train_test_split_dataset", +] diff --git a/confostate/data/datasets.py b/confostate/data/datasets.py new file mode 100644 index 0000000..93083cd --- /dev/null +++ b/confostate/data/datasets.py @@ -0,0 +1,156 @@ +"""Dataset assembly and split helpers for ML workflows.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable + +import numpy as np +import pandas as pd + + +REQUIRED_ANNOTATION_COLUMNS = {"pdb_id", "conformation"} + + +@dataclass +class DatasetBundle: + """Container for merged dataset and derived feature metadata.""" + + dataframe: pd.DataFrame + feature_columns: list[str] + label_column: str = "conformation" + + +def _require_columns( + df: pd.DataFrame, required: Iterable[str], name: str +) -> None: + missing = [c for c in required if c not in df.columns] + if missing: + raise ValueError(f"{name} is missing required columns: {missing}") + + +def _normalize_pdb_ids(series: pd.Series) -> pd.Series: + return series.astype(str).str.strip().str.upper() + + +def load_dataset( + annotations_csv: str, + features_csv: str, + family: str | None = None, + label_column: str = "conformation", + key_column: str = "pdb_id", +) -> DatasetBundle: + """Load and merge annotations with feature vectors by PDB id.""" + annotations = pd.read_csv(annotations_csv) + features = pd.read_csv(features_csv) + + _require_columns( + annotations, REQUIRED_ANNOTATION_COLUMNS, "annotations_csv" + ) + _require_columns(features, {key_column}, "features_csv") + _require_columns( + annotations, {key_column, label_column}, "annotations_csv" + ) + + annotations = annotations.copy() + features = features.copy() + + annotations[key_column] = _normalize_pdb_ids(annotations[key_column]) + features[key_column] = _normalize_pdb_ids(features[key_column]) + + if family is not None: + if "family" not in annotations.columns: + raise ValueError( + "family filter requested, but 'family' is not present in " + "annotations_csv" + ) + annotations = annotations[ + annotations["family"].astype(str) == str(family) + ] + + merged = annotations.merge( + features, on=key_column, how="inner", suffixes=("", "_feature") + ) + if merged.empty: + raise ValueError( + "Merged dataset is empty. Verify overlap between annotations and " + "features keys." + ) + + key_like = { + key_column, + label_column, + "family", + "reference", + "experimental_method", + } + feature_columns = [c for c in merged.columns if c not in key_like] + if not feature_columns: + raise ValueError("No feature columns found after merge.") + + for col in feature_columns: + merged[col] = pd.to_numeric(merged[col], errors="raise") + + return DatasetBundle( + dataframe=merged, + feature_columns=feature_columns, + label_column=label_column, + ) + + +def train_test_split_dataset( + df: pd.DataFrame, + label_column: str = "conformation", + test_size: float = 0.2, + random_state: int = 42, + stratify: bool = True, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Split dataset into train/test. + + Uses scikit-learn if available and falls back otherwise. + """ + if label_column not in df.columns: + raise ValueError( + f"Label column '{label_column}' not found in dataframe" + ) + + if not 0.0 < test_size < 1.0: + raise ValueError("test_size must be between 0 and 1") + + try: + from sklearn.model_selection import train_test_split + + stratify_values = df[label_column] if stratify else None + train_df, test_df = train_test_split( + df, + test_size=test_size, + random_state=random_state, + stratify=stratify_values, + ) + return train_df.reset_index(drop=True), test_df.reset_index(drop=True) + except Exception: + rng = np.random.default_rng(seed=random_state) + indices = np.arange(len(df)) + rng.shuffle(indices) + + n_test = max(1, int(round(len(df) * test_size))) + test_idx = set(indices[:n_test].tolist()) + + test_df = df.iloc[[i for i in range(len(df)) if i in test_idx]].copy() + train_df = df.iloc[ + [i for i in range(len(df)) if i not in test_idx] + ].copy() + return train_df.reset_index(drop=True), test_df.reset_index(drop=True) + + +def build_xy( + df: pd.DataFrame, + feature_columns: list[str], + label_column: str = "conformation", +) -> tuple[pd.DataFrame, pd.Series]: + """Extract model-ready feature matrix and labels.""" + _require_columns(df, feature_columns, "dataset") + _require_columns(df, {label_column}, "dataset") + X = df[feature_columns].copy() + y = df[label_column].copy() + return X, y diff --git a/confostate/models/__init__.py b/confostate/models/__init__.py new file mode 100644 index 0000000..5da76b4 --- /dev/null +++ b/confostate/models/__init__.py @@ -0,0 +1,15 @@ +"""Model training and evaluation utilities for ConfoState.""" + +from confostate.models.baseline import get_baseline_models +from confostate.models.evaluate import evaluate_model, write_evaluation_report +from confostate.models.registry import get_registered_model, register_model +from confostate.models.train import run_training + +__all__ = [ + "evaluate_model", + "get_baseline_models", + "get_registered_model", + "register_model", + "run_training", + "write_evaluation_report", +] diff --git a/confostate/models/annotation_table_training.py b/confostate/models/annotation_table_training.py new file mode 100644 index 0000000..c67fb59 --- /dev/null +++ b/confostate/models/annotation_table_training.py @@ -0,0 +1,168 @@ +"""Training helpers that use annotation-table columns as model inputs.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pandas as pd + +DEFAULT_NUMERIC_INPUTS = [ + "resolution_angstrom", + "year", + "opm_tilt_angle", + "opm_rotation_angle", + "opm_depth", + "opm_tm_count", +] + +DEFAULT_CATEGORICAL_INPUTS = [ + "family", + "conformation_status", + "reference", + "experimental_method", + "metadata_status", + "opm_status", +] + + +def _prepare_inputs( + df: pd.DataFrame, numeric_cols: list[str], categorical_cols: list[str] +) -> pd.DataFrame: + prepared = df.copy() + for col in numeric_cols: + prepared[col] = pd.to_numeric(prepared[col], errors="coerce") + for col in categorical_cols: + prepared[col] = prepared[col].astype(str) + return prepared + + +def train_from_annotations_table( + annotations_csv: str, + model_out: str, + metrics_out: str, + target_col: str = "conformation", + numeric_cols: list[str] | None = None, + categorical_cols: list[str] | None = None, + test_size: float = 0.2, + random_state: int = 42, +) -> dict: + """Train a baseline classifier from annotation table columns.""" + try: + import joblib + from sklearn.compose import ColumnTransformer + from sklearn.impute import SimpleImputer + from sklearn.linear_model import LogisticRegression + from sklearn.metrics import ( + accuracy_score, + classification_report, + confusion_matrix, + ) + from sklearn.model_selection import train_test_split + from sklearn.pipeline import Pipeline + from sklearn.preprocessing import OneHotEncoder + except ImportError as exc: + raise ImportError( + "scikit-learn and joblib are required. " + "Install with: pip install scikit-learn joblib" + ) from exc + + numeric = numeric_cols or DEFAULT_NUMERIC_INPUTS + categorical = categorical_cols or DEFAULT_CATEGORICAL_INPUTS + + df = pd.read_csv( + annotations_csv, + na_values=["N/A", "n/a", "NA", ""], + keep_default_na=True, + ) + + required = [target_col] + numeric + categorical + missing = [c for c in required if c not in df.columns] + if missing: + raise ValueError( + f"Missing required columns in annotations file: {missing}" + ) + + working = _prepare_inputs(df[required], numeric, categorical) + X = working[numeric + categorical] + y = working[target_col] + + if y.nunique() < 2: + raise ValueError( + "Training needs at least two target classes in the input table" + ) + + X_train, X_test, y_train, y_test = train_test_split( + X, + y, + test_size=test_size, + random_state=random_state, + stratify=y, + ) + + preprocessor = ColumnTransformer( + transformers=[ + ( + "num", + Pipeline([("imputer", SimpleImputer(strategy="median"))]), + numeric, + ), + ( + "cat", + Pipeline( + [ + ("imputer", SimpleImputer(strategy="most_frequent")), + ("onehot", OneHotEncoder(handle_unknown="ignore")), + ] + ), + categorical, + ), + ] + ) + + model = Pipeline( + [ + ("preprocessor", preprocessor), + ( + "classifier", + LogisticRegression( + max_iter=2000, + class_weight="balanced", + random_state=random_state, + ), + ), + ] + ) + + model.fit(X_train, y_train) + y_pred = model.predict(X_test) + + labels = sorted(y.unique().tolist()) + metrics = { + "annotations_csv": annotations_csv, + "input_columns": numeric + categorical, + "numeric_columns": numeric, + "categorical_columns": categorical, + "target_column": target_col, + "n_total": int(len(df)), + "n_train": int(len(X_train)), + "n_test": int(len(X_test)), + "labels": labels, + "accuracy": float(accuracy_score(y_test, y_pred)), + "classification_report": classification_report( + y_test, y_pred, output_dict=True, zero_division=0 + ), + "confusion_matrix": confusion_matrix( + y_test, y_pred, labels=labels + ).tolist(), + } + + model_path = Path(model_out) + metrics_path = Path(metrics_out) + model_path.parent.mkdir(parents=True, exist_ok=True) + metrics_path.parent.mkdir(parents=True, exist_ok=True) + + joblib.dump(model, model_path) + metrics_path.write_text(json.dumps(metrics, indent=2), encoding="utf-8") + + return metrics diff --git a/confostate/models/baseline.py b/confostate/models/baseline.py new file mode 100644 index 0000000..b4bba93 --- /dev/null +++ b/confostate/models/baseline.py @@ -0,0 +1,40 @@ +"""Baseline model definitions for conformational state classification.""" + +from __future__ import annotations + + +def get_baseline_models(random_state: int = 42) -> dict[str, object]: + """Return baseline estimators keyed by model name.""" + try: + from sklearn.ensemble import RandomForestClassifier + from sklearn.linear_model import LogisticRegression + from sklearn.svm import SVC + except ImportError as exc: + raise ImportError( + "scikit-learn is required for baseline models. " + "Install with: pip install scikit-learn" + ) from exc + + return { + "logreg": LogisticRegression( + max_iter=2000, random_state=random_state, class_weight="balanced" + ), + "random_forest": RandomForestClassifier( + n_estimators=300, + random_state=random_state, + class_weight="balanced_subsample", + ), + "svm_rbf": SVC( + C=1.0, + kernel="rbf", + gamma="scale", + probability=True, + class_weight="balanced", + ), + } + + +def train_model(estimator: object, X_train, y_train) -> object: + """Fit and return a baseline model.""" + estimator.fit(X_train, y_train) + return estimator diff --git a/confostate/models/evaluate.py b/confostate/models/evaluate.py new file mode 100644 index 0000000..cdc6028 --- /dev/null +++ b/confostate/models/evaluate.py @@ -0,0 +1,117 @@ +"""Evaluation and reporting helpers for ConfoState models.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np + + +def evaluate_model( + model, X_test, y_test, labels: list[str] | None = None +) -> dict: + """Compute standard classification metrics. + + Returns a serializable dictionary. + """ + try: + from sklearn.metrics import ( + accuracy_score, + classification_report, + confusion_matrix, + precision_recall_fscore_support, + ) + except ImportError as exc: + raise ImportError( + "scikit-learn is required for evaluation. " + "Install with: pip install scikit-learn" + ) from exc + + y_pred = model.predict(X_test) + used_labels = labels or sorted({str(v) for v in y_test}) + + accuracy = float(accuracy_score(y_test, y_pred)) + precision, recall, f1, support = precision_recall_fscore_support( + y_test, + y_pred, + labels=used_labels, + zero_division=0, + ) + + cm = confusion_matrix(y_test, y_pred, labels=used_labels) + cls_report = classification_report( + y_test, y_pred, labels=used_labels, output_dict=True, zero_division=0 + ) + + metrics = { + "accuracy": accuracy, + "labels": used_labels, + "confusion_matrix": cm.tolist(), + "per_label": { + label: { + "precision": float(precision[i]), + "recall": float(recall[i]), + "f1": float(f1[i]), + "support": int(support[i]), + } + for i, label in enumerate(used_labels) + }, + "report": cls_report, + "n_test": int(len(y_test)), + } + + if hasattr(model, "predict_proba"): + proba = model.predict_proba(X_test) + metrics["mean_confidence"] = float(np.max(proba, axis=1).mean()) + + return metrics + + +def write_evaluation_report( + metrics: dict, + output_path: str, + title: str = "ConfoState Evaluation Report", +) -> None: + """Write a compact markdown report from metrics.""" + out = Path(output_path) + out.parent.mkdir(parents=True, exist_ok=True) + + lines = [ + f"# {title}", + "", + f"- Test samples: {metrics.get('n_test', 'n/a')}", + f"- Accuracy: {metrics.get('accuracy', 0.0):.4f}", + ] + + if "mean_confidence" in metrics: + lines.append(f"- Mean confidence: {metrics['mean_confidence']:.4f}") + + lines.extend( + [ + "", + "## Per-label Metrics", + "", + "| Label | Precision | Recall | F1 | Support |", + "|---|---:|---:|---:|---:|", + ] + ) + + for label, m in metrics.get("per_label", {}).items(): + lines.append( + f"| {label} | {m['precision']:.3f} | {m['recall']:.3f} | " + f"{m['f1']:.3f} | {m['support']} |" + ) + + lines.extend( + [ + "", + "## Confusion Matrix", + "", + "```json", + json.dumps(metrics.get("confusion_matrix", []), indent=2), + "```", + ] + ) + + out.write_text("\n".join(lines), encoding="utf-8") diff --git a/confostate/models/registry.py b/confostate/models/registry.py new file mode 100644 index 0000000..e9c68b0 --- /dev/null +++ b/confostate/models/registry.py @@ -0,0 +1,72 @@ +"""Model registry helpers for tracking trained artifacts.""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from typing import Any + + +def _load_registry(path: Path) -> dict[str, Any]: + if not path.exists(): + return {"models": []} + return json.loads(path.read_text(encoding="utf-8")) + + +def _save_registry(path: Path, registry: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(registry, indent=2), encoding="utf-8") + + +def register_model( + registry_path: str, + family: str, + model_name: str, + artifact_path: str, + metrics: dict[str, Any] | None = None, + data_version: str | None = None, + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Append a model record to registry and return created record.""" + path = Path(registry_path) + registry = _load_registry(path) + + entry: dict[str, Any] = { + "family": family, + "model_name": model_name, + "artifact_path": artifact_path, + "registered_at": datetime.utcnow().isoformat() + "Z", + } + + if metrics is not None: + entry["metrics"] = metrics + if data_version is not None: + entry["data_version"] = data_version + if extra: + entry.update(extra) + + registry.setdefault("models", []).append(entry) + _save_registry(path, registry) + return entry + + +def get_registered_model( + registry_path: str, family: str +) -> dict[str, Any] | None: + """Return latest model entry for a family.""" + path = Path(registry_path) + registry = _load_registry(path) + family_entries = [ + m for m in registry.get("models", []) if m.get("family") == family + ] + if not family_entries: + return None + return family_entries[-1] + + +def list_registered_models(registry_path: str) -> list[dict[str, Any]]: + """Return all registry entries.""" + path = Path(registry_path) + registry = _load_registry(path) + return registry.get("models", []) diff --git a/confostate/models/train.py b/confostate/models/train.py new file mode 100644 index 0000000..905b04b --- /dev/null +++ b/confostate/models/train.py @@ -0,0 +1,95 @@ +"""Training pipeline helpers for baseline ConfoState models.""" + +from __future__ import annotations + +import json +import pickle +from pathlib import Path +from typing import Any + +from confostate.data.datasets import ( + build_xy, + load_dataset, + train_test_split_dataset, +) +from confostate.models.baseline import get_baseline_models, train_model +from confostate.models.evaluate import evaluate_model + + +def save_model_artifact( + model: object, out_dir: str, metadata: dict[str, Any] +) -> tuple[str, str]: + """Save model pickle and metadata JSON and return paths.""" + output = Path(out_dir) + output.mkdir(parents=True, exist_ok=True) + + model_path = output / "model.pkl" + metadata_path = output / "metadata.json" + + with model_path.open("wb") as f: + pickle.dump(model, f) + + metadata_path.write_text(json.dumps(metadata, indent=2), encoding="utf-8") + return str(model_path), str(metadata_path) + + +def run_training( + annotations_csv: str, + features_csv: str, + model_name: str, + out_dir: str, + family: str | None = None, + test_size: float = 0.2, + random_state: int = 42, +) -> dict[str, Any]: + """Run a train/eval cycle for a selected baseline model.""" + bundle = load_dataset( + annotations_csv=annotations_csv, + features_csv=features_csv, + family=family, + ) + train_df, test_df = train_test_split_dataset( + bundle.dataframe, + label_column=bundle.label_column, + test_size=test_size, + random_state=random_state, + stratify=True, + ) + + X_train, y_train = build_xy( + train_df, bundle.feature_columns, label_column=bundle.label_column + ) + X_test, y_test = build_xy( + test_df, bundle.feature_columns, label_column=bundle.label_column + ) + + models = get_baseline_models(random_state=random_state) + if model_name not in models: + available = ", ".join(sorted(models)) + raise ValueError( + f"Unknown model '{model_name}'. Available: {available}" + ) + + model = train_model(models[model_name], X_train, y_train) + metrics = evaluate_model(model, X_test, y_test) + + metadata: dict[str, Any] = { + "model_name": model_name, + "family": family, + "test_size": test_size, + "random_state": random_state, + "feature_columns": bundle.feature_columns, + "n_train": int(len(train_df)), + "n_test": int(len(test_df)), + "metrics": metrics, + } + + model_path, metadata_path = save_model_artifact(model, out_dir, metadata) + + return { + "model_path": model_path, + "metadata_path": metadata_path, + "metrics": metrics, + "feature_columns": bundle.feature_columns, + "test_dataframe": test_df, + } diff --git a/data/models/annotations_baseline_logreg.joblib b/data/models/annotations_baseline_logreg.joblib new file mode 100644 index 0000000..ab011df Binary files /dev/null and b/data/models/annotations_baseline_logreg.joblib differ diff --git a/data/models/annotations_baseline_metrics.json b/data/models/annotations_baseline_metrics.json new file mode 100644 index 0000000..e56b444 --- /dev/null +++ b/data/models/annotations_baseline_metrics.json @@ -0,0 +1,109 @@ +{ + "annotations_csv": "data/annotations/leu_t_transporters.csv.example", + "input_columns": [ + "resolution_angstrom", + "year", + "opm_tilt_angle", + "opm_rotation_angle", + "opm_depth", + "opm_tm_count", + "family", + "conformation_status", + "reference", + "experimental_method", + "metadata_status", + "opm_status" + ], + "numeric_columns": [ + "resolution_angstrom", + "year", + "opm_tilt_angle", + "opm_rotation_angle", + "opm_depth", + "opm_tm_count" + ], + "categorical_columns": [ + "family", + "conformation_status", + "reference", + "experimental_method", + "metadata_status", + "opm_status" + ], + "target_column": "conformation", + "n_total": 25, + "n_train": 20, + "n_test": 5, + "labels": [ + "IF_open", + "Intermediate", + "OF_open", + "Occluded" + ], + "accuracy": 0.2, + "classification_report": { + "IF_open": { + "precision": 0.0, + "recall": 0.0, + "f1-score": 0.0, + "support": 2.0 + }, + "Intermediate": { + "precision": 0.0, + "recall": 0.0, + "f1-score": 0.0, + "support": 1.0 + }, + "OF_open": { + "precision": 0.5, + "recall": 1.0, + "f1-score": 0.6666666666666666, + "support": 1.0 + }, + "Occluded": { + "precision": 0.0, + "recall": 0.0, + "f1-score": 0.0, + "support": 1.0 + }, + "accuracy": 0.2, + "macro avg": { + "precision": 0.125, + "recall": 0.25, + "f1-score": 0.16666666666666666, + "support": 5.0 + }, + "weighted avg": { + "precision": 0.1, + "recall": 0.2, + "f1-score": 0.13333333333333333, + "support": 5.0 + } + }, + "confusion_matrix": [ + [ + 0, + 1, + 0, + 1 + ], + [ + 0, + 0, + 0, + 1 + ], + [ + 0, + 0, + 1, + 0 + ], + [ + 0, + 0, + 1, + 0 + ] + ] +} \ No newline at end of file diff --git a/docs/USAGE.md b/docs/USAGE.md index 26263b3..9e40cf0 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -67,7 +67,45 @@ Pre-curated PDB code lists live in `data/protein_families/`: ## 5. Train and evaluate a classifier -*(To be documented as model training workflows are developed.)* +Person 3 outline scripts are available to scaffold ML workflows: + + python scripts/p3_dataset_loader.py \ + --features-csv data/features/leut_features.csv \ + --out-dir data/processed + + python scripts/p3_baseline_models.py \ + --features-csv data/features/leut_features.csv \ + --model logreg \ + --out-dir data/models/leut/logreg + + python scripts/p3_training_pipeline.py \ + --features-csv data/features/leut_features.csv \ + --models logreg random_forest svm_rbf \ + --out-dir data/models + + python scripts/p3_evaluate_reporting.py \ + --model-path data/models/leut/logreg/model.pkl \ + --test-csv data/processed/test_split.csv \ + --report-path docs/reports/phase3-eval-report.md + + python scripts/p3_model_registry.py \ + --family LeuT \ + --model-name logreg \ + --artifact-path data/models/leut/logreg/model.pkl \ + --metrics-json docs/reports/phase3-eval-metrics.json \ + --show-latest + +Train a baseline classifier from the annotation-table inputs defined in +`data/annotations/leu_t_transporters.csv.example`: + + python scripts/train_annotation_model.py + +Use a real annotation file (same schema) once available: + + python scripts/train_annotation_model.py \ + --annotations data/annotations/leu_t_transporters.csv \ + --model-out data/models/annotations_baseline_logreg.joblib \ + --metrics-out data/models/annotations_baseline_metrics.json --- diff --git a/pyproject.toml b/pyproject.toml index 9b64106..aff0fbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,8 +16,10 @@ authors = [ dependencies = [ "pandas>=1.3.0", "numpy>=1.21.0", + "scikit-learn>=1.2.0", "MDAnalysis>=2.4.0", "scipy>=1.7.0", + "joblib>=1.2.0", ] [project.optional-dependencies] diff --git a/scripts/p3_baseline_models.py b/scripts/p3_baseline_models.py new file mode 100644 index 0000000..11f0de5 --- /dev/null +++ b/scripts/p3_baseline_models.py @@ -0,0 +1,54 @@ +"""Outline script for Person 3 Task 2: baseline model training.""" + +from __future__ import annotations + +import argparse + +from confostate.models.baseline import get_baseline_models +from confostate.models.train import run_training + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Train one baseline model from merged inputs." + ) + parser.add_argument( + "--annotations-csv", default="data/annotations/leu_t_transporters.csv" + ) + parser.add_argument("--features-csv", required=True) + parser.add_argument("--family", default=None) + parser.add_argument( + "--model", + choices=["logreg", "random_forest", "svm_rbf"], + default="logreg", + ) + parser.add_argument("--out-dir", default="data/models") + parser.add_argument("--test-size", type=float, default=0.2) + parser.add_argument("--random-state", type=int, default=42) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + available = sorted( + get_baseline_models(random_state=args.random_state).keys() + ) + print(f"Available baseline models: {', '.join(available)}") + + result = run_training( + annotations_csv=args.annotations_csv, + features_csv=args.features_csv, + model_name=args.model, + out_dir=args.out_dir, + family=args.family, + test_size=args.test_size, + random_state=args.random_state, + ) + + print(f"Model artifact: {result['model_path']}") + print(f"Metadata: {result['metadata_path']}") + print(f"Accuracy: {result['metrics'].get('accuracy', 0.0):.4f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/p3_dataset_loader.py b/scripts/p3_dataset_loader.py new file mode 100644 index 0000000..e2e432b --- /dev/null +++ b/scripts/p3_dataset_loader.py @@ -0,0 +1,63 @@ +"""Outline script for Person 3 Task 1: dataset assembly and split.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from confostate.data.datasets import load_dataset, train_test_split_dataset + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build merged ML dataset and train/test splits." + ) + parser.add_argument( + "--annotations-csv", default="data/annotations/leu_t_transporters.csv" + ) + parser.add_argument("--features-csv", required=True) + parser.add_argument("--family", default=None) + parser.add_argument("--out-dir", default="data/processed") + parser.add_argument("--test-size", type=float, default=0.2) + parser.add_argument("--random-state", type=int, default=42) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + bundle = load_dataset( + annotations_csv=args.annotations_csv, + features_csv=args.features_csv, + family=args.family, + ) + + train_df, test_df = train_test_split_dataset( + bundle.dataframe, + label_column=bundle.label_column, + test_size=args.test_size, + random_state=args.random_state, + stratify=True, + ) + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + merged_path = out_dir / "merged_dataset.csv" + train_path = out_dir / "train_split.csv" + test_path = out_dir / "test_split.csv" + + bundle.dataframe.to_csv(merged_path, index=False) + train_df.to_csv(train_path, index=False) + test_df.to_csv(test_path, index=False) + + print(f"Merged dataset: {len(bundle.dataframe)} rows") + print(f"Train split: {len(train_df)} rows") + print(f"Test split: {len(test_df)} rows") + print(f"Feature count: {len(bundle.feature_columns)}") + print(f"Wrote: {merged_path}") + print(f"Wrote: {train_path}") + print(f"Wrote: {test_path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/p3_evaluate_reporting.py b/scripts/p3_evaluate_reporting.py new file mode 100644 index 0000000..226d900 --- /dev/null +++ b/scripts/p3_evaluate_reporting.py @@ -0,0 +1,68 @@ +"""Outline script for Person 3 Task 4: evaluation and report generation.""" + +from __future__ import annotations + +import argparse +import json +import pickle +from pathlib import Path + +import pandas as pd + +from confostate.models.evaluate import evaluate_model, write_evaluation_report + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate trained model and write report artifacts." + ) + parser.add_argument("--model-path", required=True) + parser.add_argument( + "--test-csv", + required=True, + help="CSV containing label and feature columns", + ) + parser.add_argument("--label-col", default="conformation") + parser.add_argument( + "--drop-cols", + nargs="*", + default=["pdb_id", "family", "reference", "experimental_method"], + ) + parser.add_argument( + "--report-path", default="docs/reports/phase3-eval-report.md" + ) + parser.add_argument( + "--metrics-json", default="docs/reports/phase3-eval-metrics.json" + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + with open(args.model_path, "rb") as handle: + model = pickle.load(handle) + + test_df = pd.read_csv(args.test_csv) + if args.label_col not in test_df.columns: + raise ValueError(f"Label column not found: {args.label_col}") + + drop_cols = [c for c in args.drop_cols if c in test_df.columns] + X_test = test_df.drop(columns=drop_cols + [args.label_col]) + y_test = test_df[args.label_col] + + metrics = evaluate_model(model, X_test, y_test) + + write_evaluation_report(metrics, args.report_path) + + metrics_path = Path(args.metrics_json) + metrics_path.parent.mkdir(parents=True, exist_ok=True) + metrics_path.write_text(json.dumps(metrics, indent=2), encoding="utf-8") + + print(f"Accuracy: {metrics.get('accuracy', 0.0):.4f}") + print(f"Report: {args.report_path}") + print(f"Metrics JSON: {args.metrics_json}") + + +if __name__ == "__main__": + main() diff --git a/scripts/p3_model_registry.py b/scripts/p3_model_registry.py new file mode 100644 index 0000000..c7d25d5 --- /dev/null +++ b/scripts/p3_model_registry.py @@ -0,0 +1,59 @@ +"""Outline script for Person 3 Task 5: model registry update.""" + +from __future__ import annotations + +import argparse +import json + +from confostate.models.registry import ( + get_registered_model, + list_registered_models, + register_model, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Register and inspect model artifacts." + ) + parser.add_argument("--registry-path", default="data/models/registry.json") + parser.add_argument("--family", required=True) + parser.add_argument("--model-name", required=True) + parser.add_argument("--artifact-path", required=True) + parser.add_argument("--metrics-json", default=None) + parser.add_argument("--data-version", default=None) + parser.add_argument("--show-latest", action="store_true") + parser.add_argument("--list", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + metrics = None + if args.metrics_json: + with open(args.metrics_json, "r", encoding="utf-8") as handle: + metrics = json.load(handle) + + entry = register_model( + registry_path=args.registry_path, + family=args.family, + model_name=args.model_name, + artifact_path=args.artifact_path, + metrics=metrics, + data_version=args.data_version, + ) + print("Registered model entry:") + print(json.dumps(entry, indent=2)) + + if args.show_latest: + latest = get_registered_model(args.registry_path, args.family) + print("Latest for family:") + print(json.dumps(latest, indent=2)) + + if args.list: + print("All registry entries:") + print(json.dumps(list_registered_models(args.registry_path), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/p3_training_pipeline.py b/scripts/p3_training_pipeline.py new file mode 100644 index 0000000..880649e --- /dev/null +++ b/scripts/p3_training_pipeline.py @@ -0,0 +1,52 @@ +"""Outline script for Person 3 Task 3: multi-model training pipeline.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from confostate.models.train import run_training + + +DEFAULT_MODELS = ["logreg", "random_forest", "svm_rbf"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run baseline training pipeline across selected models." + ) + parser.add_argument( + "--annotations-csv", default="data/annotations/leu_t_transporters.csv" + ) + parser.add_argument("--features-csv", required=True) + parser.add_argument("--family", default=None) + parser.add_argument("--models", nargs="+", default=DEFAULT_MODELS) + parser.add_argument("--out-dir", default="data/models") + parser.add_argument("--test-size", type=float, default=0.2) + parser.add_argument("--random-state", type=int, default=42) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + root = Path(args.out_dir) + root.mkdir(parents=True, exist_ok=True) + + for model_name in args.models: + model_dir = root / (args.family or "all") / model_name + print(f"Training: {model_name}") + result = run_training( + annotations_csv=args.annotations_csv, + features_csv=args.features_csv, + model_name=model_name, + out_dir=str(model_dir), + family=args.family, + test_size=args.test_size, + random_state=args.random_state, + ) + print(f" model: {result['model_path']}") + print(f" accuracy: {result['metrics'].get('accuracy', 0.0):.4f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/train_annotation_model.py b/scripts/train_annotation_model.py new file mode 100644 index 0000000..f1d8c8b --- /dev/null +++ b/scripts/train_annotation_model.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +"""Train a baseline model directly from annotation-table input columns. + +By default this script uses the columns defined in +`data/annotations/leu_t_transporters.csv.example`. +""" + +from __future__ import annotations + +import argparse + +from confostate.models.annotation_table_training import ( + train_from_annotations_table, +) + +DEFAULT_ANNOTATIONS = "data/annotations/leu_t_transporters.csv.example" +DEFAULT_MODEL_OUT = "data/models/annotations_baseline_logreg.joblib" +DEFAULT_METRICS_OUT = "data/models/annotations_baseline_metrics.json" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Train baseline model from annotations-table columns." + ) + parser.add_argument( + "--annotations", + default=DEFAULT_ANNOTATIONS, + help=f"Path to annotations CSV (default: {DEFAULT_ANNOTATIONS})", + ) + parser.add_argument( + "--model-out", + default=DEFAULT_MODEL_OUT, + help=f"Output model path (default: {DEFAULT_MODEL_OUT})", + ) + parser.add_argument( + "--metrics-out", + default=DEFAULT_METRICS_OUT, + help=f"Output metrics JSON path (default: {DEFAULT_METRICS_OUT})", + ) + parser.add_argument( + "--target-col", + default="conformation", + help="Target column for labels (default: conformation)", + ) + parser.add_argument( + "--test-size", + type=float, + default=0.2, + help="Test split fraction (default: 0.2)", + ) + parser.add_argument( + "--random-state", + type=int, + default=42, + help="Random seed for split/model (default: 42)", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + metrics = train_from_annotations_table( + annotations_csv=args.annotations, + model_out=args.model_out, + metrics_out=args.metrics_out, + target_col=args.target_col, + test_size=args.test_size, + random_state=args.random_state, + ) + + print( + "Trained on " + f"{metrics['n_train']} rows, tested on {metrics['n_test']} rows" + ) + print(f"Accuracy: {metrics['accuracy']:.4f}") + print(f"Model: {args.model_out}") + print(f"Metrics: {args.metrics_out}") + + +if __name__ == "__main__": + main()