Config

Configuration data structures and loaders for rubric and role definitions.

The pipeline requires two JSON config files

• config/lens_rubric.json: defines the 8 evaluation dimensions
• config/roles.json: defines the 3 clinical roles with per-dimension weight vectors (w_prior) and optional LLM prompt profile paths

This module parses those files into frozen dataclasses used throughout the scoring and orchestration layers.

Dimension dataclass

A single rubric evaluation dimension (e.g. "factual_accuracy").

Source code in src/grading_pipeline/config.py

@dataclass(frozen=True)
class Dimension:
    """A single rubric evaluation dimension (e.g. "factual_accuracy")."""

    id: str
    name: str
    definition: str
    evaluation_focus: str

RoleProfile dataclass

A clinical role's configuration: identity, weights, and LLM profile.

Attributes:

Name	Type	Description
id	str	Machine-readable identifier (e.g. "physician", "triage_nurse").
name	str	Human-readable role name.
persona	str	One-line persona description used in LLM prompts.
w_prior	Dict[str, float]	Per-dimension importance weights, each in [0, 1].
prompt_profile	Dict[str, Any]	Optional LLM-specific scoring profile loaded from a separate JSON file (empty dict if not provided).

Source code in src/grading_pipeline/config.py

@dataclass(frozen=True)
class RoleProfile:
    """A clinical role's configuration: identity, weights, and LLM profile.

    Attributes:
        id: Machine-readable identifier (e.g. "physician", "triage_nurse").
        name: Human-readable role name.
        persona: One-line persona description used in LLM prompts.
        w_prior: Per-dimension importance weights, each in [0, 1].
        prompt_profile: Optional LLM-specific scoring profile loaded from
            a separate JSON file (empty dict if not provided).
    """

    id: str
    name: str
    persona: str
    w_prior: Dict[str, float]
    prompt_profile: Dict[str, Any]

Rubric dataclass

The full evaluation rubric containing all scoring dimensions.

Source code in src/grading_pipeline/config.py

@dataclass(frozen=True)
class Rubric:
    """The full evaluation rubric containing all scoring dimensions."""

    rubric_id: str
    dimensions: List[Dimension]

    @property
    def dimension_ids(self) -> List[str]:
        """Return ordered list of dimension ID strings."""
        return [d.id for d in self.dimensions]

dimension_ids property

Return ordered list of dimension ID strings.

load_roles(path, dimension_ids)

Load role definitions from JSON, validating weights against the rubric.

Parameters:

Name	Type	Description	Default
path	str | Path	Path to roles.json.	required
dimension_ids	List[str]	Expected dimension IDs from the rubric, used to detect missing or extraneous weight keys.	required

Returns:

Type	Description
List[RoleProfile]	List of RoleProfile instances, one per role in the config.

Raises:

Type	Description
ValueError	On missing/extra dimension weights or invalid weight values.

Source code in src/grading_pipeline/config.py

def load_roles(path: str | Path, dimension_ids: List[str]) -> List[RoleProfile]:
    """Load role definitions from JSON, validating weights against the rubric.

    Args:
        path: Path to ``roles.json``.
        dimension_ids: Expected dimension IDs from the rubric, used to
            detect missing or extraneous weight keys.

    Returns:
        List of ``RoleProfile`` instances, one per role in the config.

    Raises:
        ValueError: On missing/extra dimension weights or invalid weight values.
    """
    roles_path = Path(path)
    data = json.loads(roles_path.read_text())

    roles: List[RoleProfile] = []
    for item in data["roles"]:
        w_prior = item["w_prior"]
        missing = set(dimension_ids) - set(w_prior.keys())
        extra = set(w_prior.keys()) - set(dimension_ids)
        if missing:
            raise ValueError(
                f"Role {item['id']} missing weights for: {sorted(missing)}"
            )
        if extra:
            raise ValueError(
                f"Role {item['id']} has unknown dimensions: {sorted(extra)}"
            )

        normalized_weights = {k: float(v) for k, v in w_prior.items()}
        _validate_weights(item["id"], normalized_weights)

        roles.append(
            RoleProfile(
                id=item["id"],
                name=item["name"],
                persona=item["persona"],
                w_prior=normalized_weights,
                prompt_profile=_load_prompt_profile(roles_path, item),
            )
        )

    return roles

load_rubric(path)

Load and parse a rubric JSON file into a Rubric instance.

Source code in src/grading_pipeline/config.py

def load_rubric(path: str | Path) -> Rubric:
    """Load and parse a rubric JSON file into a ``Rubric`` instance."""
    data = json.loads(Path(path).read_text())
    dims = [
        Dimension(
            id=item["id"],
            name=item["name"],
            definition=item["definition"],
            evaluation_focus=item["evaluation_focus"],
        )
        for item in data["dimensions"]
    ]
    return Rubric(rubric_id=data["rubric_id"], dimensions=dims)