Generation

Modules:

Name	Description
generation_steps
model_config

Classes:

Name	Description
GenerationSteps	A class for specifying generation steps for LLMs, including prompting.
ModelConfig	Configuration for a model to be used in an experiment.
ModelRecord	A record identifying a model.

GenerationSteps dataclass

A class for specifying generation steps for LLMs, including prompting.

Source code in evalsense/generation/generation_steps.py

@dataclass
class GenerationSteps:
    """A class for specifying generation steps for LLMs, including prompting."""

    name: str
    steps: Solver | list[Solver]

ModelConfig dataclass

Configuration for a model to be used in an experiment.

Attributes:

Name	Type	Description
name	str	Returns the name of the model.
record	ModelRecord	Returns a record of the model configuration.

Source code in evalsense/generation/model_config.py

@dataclass
class ModelConfig:
    """Configuration for a model to be used in an experiment."""

    model: str | Model
    model_args: dict[str, Any] = field(default_factory=dict)
    generation_args: GenerateConfigArgs = field(default_factory=GenerateConfigArgs)

    @property
    def name(self) -> str:
        """Returns the name of the model."""
        if isinstance(self.model, str):
            return self.model
        return self.model.name

    @property
    def record(self) -> ModelRecord:
        """Returns a record of the model configuration."""
        # Remove arguments not directly affecting the used model or generation
        # procedure, so that we can match equivalent records.
        filtered_model_args = {
            k: v
            for k, v in self.model_args.items()
            if k not in {"device", "gpu_memory_utilization", "download_dir"}
        }
        filtered_generation_args = {
            k: v
            for k, v in self.generation_args.items()
            if k not in {"max_connections"}
        }

        return ModelRecord(
            name=self.name,
            model_args_json=json.dumps(
                filtered_model_args,
                default=str,
                sort_keys=True,
                ensure_ascii=True,
            ),
            generation_args_json=json.dumps(
                filtered_generation_args,
                default=str,
                sort_keys=True,
                ensure_ascii=True,
            ),
        )

name property

name: str

Returns the name of the model.

record property

record: ModelRecord

Returns a record of the model configuration.

ModelRecord

Bases: BaseModel

A record identifying a model.

Attributes:

Name	Type	Description
name	str	The name of the model.
model_args_json	str	The model arguments as a JSON string.
generation_args_json	str	The generation arguments as a JSON string.

Methods:

Name	Description
__eq__	Checks if this record is equal to another record.
__hash__	Returns a hash of the record.
__lt__	Checks if this record is less than another record.

Source code in evalsense/generation/model_config.py

@total_ordering
class ModelRecord(BaseModel, frozen=True):
    """A record identifying a model.

    Attributes:
        name (str): The name of the model.
        model_args_json (str): The model arguments as a JSON string.
        generation_args_json (str): The generation arguments as a JSON string.
    """

    # We need to use JSON strings here to keep the record hashable.
    name: str
    model_args_json: str = "{}"
    generation_args_json: str = "{}"

    def __eq__(self, other: object) -> bool:
        """Checks if this record is equal to another record.

        Args:
            other (object): The other record to compare with.

        Returns:
            bool: True if the records are equal, False otherwise.
        """
        if not isinstance(other, ModelRecord) or type(self) is not type(other):
            return NotImplemented
        return (
            self.name == other.name
            and self.model_args_json == other.model_args_json
            and self.generation_args_json == other.generation_args_json
        )

    def __lt__(self, other: object) -> bool:
        """Checks if this record is less than another record.

        Args:
            other (object): The other record to compare with.

        Returns:
            bool: True if this record is less than the other, False otherwise.
        """
        if not isinstance(other, ModelRecord) or type(self) is not type(other):
            return NotImplemented
        return (self.name, self.model_args_json, self.generation_args_json) < (
            other.name,
            other.model_args_json,
            other.generation_args_json,
        )

    def __hash__(self) -> int:
        """Returns a hash of the record.

        Returns:
            int: The hash of the record.
        """
        return hash((self.name, self.model_args_json, self.generation_args_json))

__eq__

__eq__(other: object) -> bool

Checks if this record is equal to another record.

Parameters:

Name	Type	Description	Default
other	object	The other record to compare with.	required

Returns:

Name	Type	Description
bool	bool	True if the records are equal, False otherwise.

Source code in evalsense/generation/model_config.py

def __eq__(self, other: object) -> bool:
    """Checks if this record is equal to another record.

    Args:
        other (object): The other record to compare with.

    Returns:
        bool: True if the records are equal, False otherwise.
    """
    if not isinstance(other, ModelRecord) or type(self) is not type(other):
        return NotImplemented
    return (
        self.name == other.name
        and self.model_args_json == other.model_args_json
        and self.generation_args_json == other.generation_args_json
    )

__hash__

__hash__() -> int

Returns a hash of the record.

Returns:

Name	Type	Description
int	int	The hash of the record.

Source code in evalsense/generation/model_config.py

def __hash__(self) -> int:
    """Returns a hash of the record.

    Returns:
        int: The hash of the record.
    """
    return hash((self.name, self.model_args_json, self.generation_args_json))

__lt__

__lt__(other: object) -> bool

Checks if this record is less than another record.

Parameters:

Name	Type	Description	Default
other	object	The other record to compare with.	required

Returns:

Name	Type	Description
bool	bool	True if this record is less than the other, False otherwise.

Source code in evalsense/generation/model_config.py

def __lt__(self, other: object) -> bool:
    """Checks if this record is less than another record.

    Args:
        other (object): The other record to compare with.

    Returns:
        bool: True if this record is less than the other, False otherwise.
    """
    if not isinstance(other, ModelRecord) or type(self) is not type(other):
        return NotImplemented
    return (self.name, self.model_args_json, self.generation_args_json) < (
        other.name,
        other.model_args_json,
        other.generation_args_json,
    )