prepare_data

Prepare data for VGG models.

Classes:

Name	Description
VGGFaceHumanjudgmentDataset	Dataset for the VGG-Face model variant, adapted for human similarity judgments.
VGGMultiViewDataset	Dataset for the VGG-Multi-View-Face model.

Functions:

Name	Description
get_multi_view_data	Get the multi-view data.
load_image_for_model	Load an image for the VGG model.
prepare_data_for_human_judgment_model	Prepare data for the VGG-Face-model for human similarity judgments.
prepare_data_for_multi_view_model	Prepare data for the multi-view model.
revert_model_image	Revert a model-input-image to its original form.

VGGFaceHumanjudgmentDataset

VGGFaceHumanjudgmentDataset(
    session: str,
    frozen_core: bool,
    data_mode: str = "2d-original",
    last_core_layer: str | None = None,
    dtype: dtype = float32,
    size: int | None = None,
    exclusive_gender_trials: str | None = None,
    heads: list[int] | ndarray[int] | int | None = None,
    **kwargs
)

Bases: Dataset

Dataset for the VGG-Face model variant, adapted for human similarity judgments.

Initialize the VGGFaceHumanjudgmentDataset.

Methods:

Name	Description
display_triplet	Display a triplet of images.

Attributes:

Name	Type	Description
current_index		Return the current index.
data_mode		Return the data mode.
exclusive_gender_trials	str | None	Return the exclusive_gender_trials configuration.
last_core_layer		Return the cut layer of the VGG core model.
session_data		Return the session data.
vgg_core_output		Return the VGG core output.

Source code in code/facesim3d/modeling/VGG/prepare_data.py

def __init__(
    self,
    session: str,
    frozen_core: bool,
    data_mode: str = "2d-original",
    last_core_layer: str | None = None,
    dtype: torch.dtype = torch.float32,
    size: int | None = None,
    exclusive_gender_trials: str | None = None,
    heads: list[int] | np.ndarray[int] | int | None = None,
    **kwargs,
) -> None:
    """Initialize the `VGGFaceHumanjudgmentDataset`."""
    self.session = session
    self._size = size
    self.exclusive_gender_trials = exclusive_gender_trials
    self._heads = heads
    self.session_data = read_trial_results_of_session(session=session, clean_trials=True, verbose=False)[
        ["head1", "head2", "head3", "head_odd"]
    ].astype(int)
    self.frozen_core = frozen_core
    self._vgg_core_output = None
    self.data_mode = data_mode.lower()
    self.last_core_layer = last_core_layer
    self._suffix_data_mode = (
        "original" if "orig" in self.data_mode else "3D-recon" if "recon" in self.data_mode else "3D-persp"
    )
    self.dtype = dtype
    self._subtract_mean = kwargs.pop("subtract_mean", True)
    self._current_index = None

current_index property

current_index

Return the current index.

data_mode property writable

data_mode

Return the data mode.

exclusive_gender_trials property writable

exclusive_gender_trials: str | None

Return the exclusive_gender_trials configuration.

last_core_layer property writable

last_core_layer

Return the cut layer of the VGG core model.

session_data property writable

session_data

Return the session data.

vgg_core_output property

vgg_core_output

Return the VGG core output.

display_triplet

display_triplet(
    idx: int | Tensor,
    as_seen_by_model: bool = True,
    verbose: bool = False,
) -> None

Display a triplet of images.

Source code in code/facesim3d/modeling/VGG/prepare_data.py

def display_triplet(self, idx: int | torch.Tensor, as_seen_by_model: bool = True, verbose: bool = False) -> None:
    """Display a triplet of images."""
    if torch.is_tensor(idx):
        idx = idx.tolist().pop()

    img1, img2, img3, _, _ = self[idx].values()  # _, _ == choice, idx
    faces_imgs = [img1, img2, img3]
    min_val = min(img.min() for img in faces_imgs)
    max_val = max(img.max() for img in faces_imgs)
    h1, h2, h3, odd = self.session_data.iloc[idx].to_numpy()
    faces = [f"Head{h}" for h in [h1, h2, h3]]

    choice_side = [h1, h2, h3].index(odd)

    # Display faces
    title = f"Session: {self.session} | {self.data_mode} | as seen by model: {as_seen_by_model} | idx: {idx}"
    color = "darkorange" if self.frozen_core and as_seen_by_model else "royalblue"

    r, c = 12, 4
    if self.frozen_core:
        x, y = dims_to_rectangularize(len(img1))
        c = round(r / x * y) - 1

    fig, axs = plt.subplots(nrows=1, ncols=3, sharex=True, sharey=True, num=title, figsize=(r, c))
    for i, ax in enumerate(axs.flatten()):
        if as_seen_by_model:
            if self.frozen_core:
                img = rectangularize_1d_array(arr=faces_imgs[i], wide=False)
                ax.imshow(img, cmap="seismic", vmin=min_val, vmax=max_val)

            else:
                img = faces_imgs[i].permute(1, 2, 0).to("cpu").numpy()
                img = (img - img.min()) / (img.max() - img.min()) * 255
                img = img.astype(np.uint8)
                img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
                ax.imshow(Image.fromarray(img))
        else:
            ax.imshow(
                display_face(
                    head_id=faces[i], data_mode=self.data_mode, interactive=False, show=False, verbose=verbose
                )
            )
        ax.set_title(faces[i], color=color if i == choice_side else "black")
        if i == choice_side:
            for spine in ax.spines.values():
                spine.set_edgecolor(color)
                spine.set_linewidth(2)
            ax.set_xticks([])
            ax.set_yticks([])
        else:
            ax.axis("off")
    fig.suptitle(title)
    fig.tight_layout()
    fig.show()

VGGMultiViewDataset

VGGMultiViewDataset(
    frozen_core: bool,
    last_core_layer: str | None = None,
    dtype: dtype = float32,
    heads: list[int] | ndarray[int] | int | None = None,
    **kwargs
)

Bases: Dataset

Dataset for the VGG-Multi-View-Face model.

Initialize the VGGMultiViewDataset.

Methods:

Name	Description
display_image	Display face images with a specific angle.

Attributes:

Name	Type	Description
current_index		Return the current index.
last_core_layer		Return cut layer of the VGG core model.
multi_view_data		Return the session data.
n_unique_faces		Return the number of unique faces.
vgg_core_output		Return the VGG core output.

Source code in code/facesim3d/modeling/VGG/prepare_data.py

def __init__(
    self,
    frozen_core: bool,
    last_core_layer: str | None = None,
    dtype: torch.dtype = torch.float32,
    heads: list[int] | np.ndarray[int] | int | None = None,
    **kwargs,
) -> None:
    """Initialize the `VGGMultiViewDataset`."""
    self._heads = heads
    self.multi_view_data = get_multi_view_data()
    self.frozen_core = frozen_core
    self._vgg_core_output = None
    self.last_core_layer = last_core_layer
    self._suffix_data_mode = "3D-persp"
    self.dtype = dtype
    self._subtract_mean = kwargs.pop("subtract_mean", True)
    self._current_index = None

current_index property

current_index

Return the current index.

last_core_layer property writable

last_core_layer

Return cut layer of the VGG core model.

multi_view_data property writable

multi_view_data

Return the session data.

n_unique_faces property

n_unique_faces

Return the number of unique faces.

vgg_core_output property

vgg_core_output

Return the VGG core output.

display_image

display_image(
    idx: int | Tensor,
    as_seen_by_model: bool = True,
    verbose: bool = False,
) -> None

Display face images with a specific angle.

Source code in code/facesim3d/modeling/VGG/prepare_data.py

def display_image(self, idx: int | torch.Tensor, as_seen_by_model: bool = True, verbose: bool = False) -> None:
    """Display face images with a specific angle."""
    if torch.is_tensor(idx):
        idx = idx.tolist().pop()

    face_img, head_nr, angle, _ = self[idx].values()  # _ == idx
    face = f"Head{head_nr}"

    # Display faces
    title = f"3d-perspectives | angle: {angle}° | as seen by model: {as_seen_by_model} | idx: {idx}"

    r, c = 10, 8
    if self.frozen_core:
        x, y = dims_to_rectangularize(len(face_img))
        c = round(r / x * y) - 1

    fig, ax = plt.subplots(nrows=1, ncols=1, sharex=True, sharey=True, num=title, figsize=(r, c))

    if as_seen_by_model:
        if self.frozen_core:
            img = rectangularize_1d_array(arr=face_img, wide=False)
            ax.imshow(img, cmap="seismic", vmin=face_img.min(), vmax=face_img.max())

        else:
            img = face_img.permute(1, 2, 0).to("cpu").numpy()
            img = (img - img.min()) / (img.max() - img.min()) * 255
            img = img.astype(np.uint8)
            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            ax.imshow(Image.fromarray(img))
    else:
        ax.imshow(
            display_face(
                head_id=face,
                data_mode="3d-perspectives",
                angle=angle,
                interactive=False,
                show=False,
                verbose=verbose,
            )
        )
    ax.set_title(face, color="black")
    ax.axis("off")
    fig.suptitle(title)
    fig.tight_layout()
    fig.show()

get_multi_view_data cached

get_multi_view_data() -> DataFrame

Get the multi-view data.

Source code in code/facesim3d/modeling/VGG/prepare_data.py

@cache
def get_multi_view_data() -> pd.DataFrame:
    """Get the multi-view data."""
    mv_tab = pd.DataFrame(columns=["head_nr", "head_idx", "angle", "img_path"])
    for head_nr in range(1, params.main.n_faces + 1):
        head_idx = head_nr_to_index(head_id=head_nr)
        for path_to_image in Path(paths.data.cfd.faceviews).glob(f"head-{head_idx:03d}_*.png"):
            angle = path_to_image.stem.split("_")[1].split("angle-")[-1]
            mv_tab.loc[len(mv_tab)] = (head_nr, head_idx, angle, path_to_image)

    # Sort table by head_nr and angle
    def adapt_angle(_angle: str) -> str:
        """Transform a given angle to a sortable str object."""
        if _angle != "frontal":
            _angle = f"{float(_angle):06.2f}"
        return _angle

    mv_tab["angle_sortable"] = mv_tab.angle.map(adapt_angle)
    mv_tab = mv_tab.sort_values(by=["head_nr", "angle_sortable"], ascending=[True, True])
    return mv_tab.reset_index(drop=True)

load_image_for_model

load_image_for_model(
    image_path: str | Path,
    dtype: float64,
    subtract_mean: bool = True,
) -> Tensor

Load an image for the VGG model.

Source code in code/facesim3d/modeling/VGG/prepare_data.py

def load_image_for_model(image_path: str | Path, dtype: torch.float64, subtract_mean: bool = True) -> torch.Tensor:
    """Load an image for the `VGG` model."""
    image = cv2.imread(str(image_path))
    image = cv2.resize(image, dsize=(224, 224))
    image = torch.Tensor(image).permute(2, 0, 1).view(1, 3, 224, 224).to(dtype)
    if subtract_mean:
        # this subtraction should be the average pixel value of the training set of the original VGGFace
        image -= torch.Tensor(np.array([129.1863, 104.7624, 93.5940])).to(dtype).view(1, 3, 1, 1)
    return image

prepare_data_for_human_judgment_model

prepare_data_for_human_judgment_model(
    session: str,
    frozen_core: bool,
    data_mode: str,
    last_core_layer: str | None = None,
    split_ratio: tuple = (0.7, 0.15, 0.15),
    batch_size: int = 1,
    shuffle: bool = True,
    num_workers: int = 0,
    dtype: dtype = float32,
    size: int | None = None,
    exclusive_gender_trials: str | None = None,
    heads: list[int] | ndarray[int] | int | None = None,
    **kwargs
) -> tuple[DataLoader, DataLoader, DataLoader]

Prepare data for the VGG-Face-model for human similarity judgments.

Split the data into a train, validation and test set.

Parameters:

Name	Type	Description	Default
session	str	'2D' OR '3D'	required
frozen_core	bool	prepare data for frozen VGG core or not	required
data_mode	str	use "2d-original", "3d-reconstructions", or "3d-perspectives" as input images	required
last_core_layer	str | None	must be given if frozen_core is True	None
split_ratio	tuple	ratio of train, validation and test set	(0.7, 0.15, 0.15)
batch_size	int	batch size for dataloader	1
shuffle	bool	shuffle data	True
num_workers	int	number of workers for dataloader	0
dtype	dtype	data type for images	float32
size	int | None	optionally define total size of data (which then gets split)	None
exclusive_gender_trials	str | None	use exclusive gender trials ['female' OR 'male'], OR None for all samples.	None
heads	list[int] | ndarray[int] | int | None	optionally define subset of data, provide a list of head IDs or total number of heads IDs	None

Returns:

Type	Description
tuple[DataLoader, DataLoader, DataLoader]	train_dataloader, validation_dataloader, test_dataloader

Source code in code/facesim3d/modeling/VGG/prepare_data.py

def prepare_data_for_human_judgment_model(
    session: str,
    frozen_core: bool,
    data_mode: str,
    last_core_layer: str | None = None,
    split_ratio: tuple = (0.7, 0.15, 0.15),
    batch_size: int = 1,
    shuffle: bool = True,
    num_workers: int = 0,
    dtype: torch.dtype = torch.float32,
    size: int | None = None,
    exclusive_gender_trials: str | None = None,
    heads: list[int] | np.ndarray[int] | int | None = None,
    **kwargs,
) -> tuple[DataLoader, DataLoader, DataLoader]:
    """
    Prepare data for the `VGG-Face`-model for human similarity judgments.

    Split the data into a train, validation and test set.

    :param session: '2D' OR '3D'
    :param frozen_core: prepare data for frozen VGG core or not
    :param data_mode: use "2d-original", "3d-reconstructions", or "3d-perspectives" as input images
    :param last_core_layer: must be given if frozen_core is True
    :param split_ratio: ratio of train, validation and test set
    :param batch_size: batch size for dataloader
    :param shuffle: shuffle data
    :param num_workers: number of workers for dataloader
    :param dtype: data type for images
    :param size: optionally define total size of data (which then gets split)
    :param exclusive_gender_trials: use exclusive gender trials ['female' OR 'male'], OR None for all samples.
    :param heads: optionally define subset of data, provide a list of head IDs or total number of heads IDs
    :return: train_dataloader, validation_dataloader, test_dataloader
    """
    # Load all data
    all_data = VGGFaceHumanjudgmentDataset(
        session=session,
        frozen_core=frozen_core,
        data_mode=data_mode,
        last_core_layer=last_core_layer,
        dtype=dtype,
        size=size,
        exclusive_gender_trials=exclusive_gender_trials,
        heads=heads,
        **kwargs,
    )

    # Split into train, validation and test set
    if sum(split_ratio) != 1.0:
        msg = "Split ratio must sum up to 1."
        raise ValueError(msg)
    cprint(
        string="Splitting data into {:.0%} training, {:.0%} validation & {:.0%} test set ...".format(*split_ratio),
        col="b",
    )
    training_size = int(split_ratio[0] * len(all_data))
    validation_size = int(split_ratio[1] * len(all_data))
    test_size = len(all_data) - training_size - validation_size
    train_data, val_data, test_data = random_split(
        dataset=all_data, lengths=[training_size, validation_size, test_size]
    )

    # Create dataloaders
    train_dataloader = (
        DataLoader(train_data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
        if training_size > 0
        else None
    )
    validation_dataloader = (
        DataLoader(val_data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
        if validation_size > 0
        else None
    )
    test_dataloader = (
        DataLoader(test_data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
        if test_size > 0
        else None
    )

    return train_dataloader, validation_dataloader, test_dataloader

prepare_data_for_multi_view_model

prepare_data_for_multi_view_model(
    frozen_core: bool,
    last_core_layer: str | None = None,
    split_ratio: tuple = (0.8, 0.2, 0.0),
    batch_size: int = 1,
    shuffle: bool = True,
    num_workers: int = 0,
    dtype: dtype = float32,
    heads: list[int] | ndarray[int] | int | None = None,
    **kwargs
) -> tuple[DataLoader, DataLoader, DataLoader]

Prepare data for the multi-view model.

Split the data into a train, validation and test set.

Parameters:

Name	Type	Description	Default
frozen_core	bool	prepare data for frozen VGG core or not	required
last_core_layer	str | None	must be given if frozen_core is True	None
split_ratio	tuple	ratio of train, validation and test set. The test set always contains the frontal views of faces. If > (..., ..., 0.) take also more views into the test set.	(0.8, 0.2, 0.0)
batch_size	int	batch size for dataloader	1
shuffle	bool	shuffle data	True
num_workers	int	number of workers for the dataloader	0
dtype	dtype	data type for images	float32
heads	list[int] | ndarray[int] | int | None	optionally define subset of data, provide a list of head IDs or total number of heads IDs	None

Returns:

Type	Description
tuple[DataLoader, DataLoader, DataLoader]	train_dataloader, validation_dataloader, test_dataloader

Source code in code/facesim3d/modeling/VGG/prepare_data.py

def prepare_data_for_multi_view_model(
    frozen_core: bool,
    last_core_layer: str | None = None,
    split_ratio: tuple = (0.8, 0.2, 0.0),
    batch_size: int = 1,
    shuffle: bool = True,
    num_workers: int = 0,
    dtype: torch.dtype = torch.float32,
    heads: list[int] | np.ndarray[int] | int | None = None,
    **kwargs,
) -> tuple[DataLoader, DataLoader, DataLoader]:
    """
    Prepare data for the multi-view model.

    Split the data into a train, validation and test set.

    :param frozen_core: prepare data for frozen VGG core or not
    :param last_core_layer: must be given if frozen_core is True
    :param split_ratio: ratio of train, validation and test set.
                        The test set always contains the frontal views of faces.
                        If > (..., ..., 0.) take also more views into the test set.
    :param batch_size: batch size for dataloader
    :param shuffle: shuffle data
    :param num_workers: number of workers for the dataloader
    :param dtype: data type for images
    :param heads: optionally define subset of data, provide a list of head IDs or total number of heads IDs
    :return: train_dataloader, validation_dataloader, test_dataloader
    """
    # Load all data
    all_data = VGGMultiViewDataset(
        frozen_core=frozen_core,
        last_core_layer=last_core_layer,
        dtype=dtype,
        heads=heads,
        **kwargs,
    )

    # Split into train, validation and test set
    if not np.allclose(sum(split_ratio), 1):
        msg = "Split ratio must sum up to 1."
        raise ValueError(msg)

    if max(split_ratio) > 1.0:
        msg = "split_ratio elements must be in in [0., 1]."
        raise ValueError(msg)

    cprint(
        string="Splitting data into {:.0%} training, {:.0%} validation & {:.0%} test set ...".format(*split_ratio),
        col="b",
    )

    # Splits are done within face ID's. At least all frontal views are in the test set.
    n_images_per_face = all_data.multi_view_data.head_nr.value_counts().max()  # == 33 (including frontal view)
    training_size_per_face = int(split_ratio[0] * (n_images_per_face - 1))  # -1 for the frontal view
    if split_ratio[2] == 0.0:
        validation_size_per_face = n_images_per_face - 1 - training_size_per_face
    else:
        validation_size_per_face = int(split_ratio[1] * (n_images_per_face - 1))
    test_size_per_face = n_images_per_face - training_size_per_face - validation_size_per_face

    training_indices = []
    validation_indices = []
    test_indices = all_data.multi_view_data[all_data.multi_view_data.angle == "frontal"].index.tolist()
    for _head_nr, data_head_nr in all_data.multi_view_data.groupby("head_nr"):
        test_indices_for_head = data_head_nr[data_head_nr.angle == "frontal"].index.tolist()
        train_indices_for_head = (
            data_head_nr[data_head_nr.angle != "frontal"].sample(training_size_per_face, replace=False).index.tolist()
        )
        training_indices += train_indices_for_head
        val_indices_for_head = (
            data_head_nr.drop(index=train_indices_for_head + test_indices_for_head)
            .sample(validation_size_per_face, replace=False)
            .index.tolist()
        )
        validation_indices += val_indices_for_head

        test_indices_for_head += (
            data_head_nr.drop(index=train_indices_for_head + val_indices_for_head + test_indices_for_head)
            .sample(test_size_per_face - 1, replace=False)
            .index.tolist()
        )
        # Add test_indices_for_head to test_indices if not already in there
        test_indices += [idx for idx in test_indices_for_head if idx not in test_indices]

    assert set(training_indices) & set(validation_indices) & set(test_indices) == set()  # noqa: S101
    assert len(training_indices) + len(validation_indices) + len(test_indices) == len(all_data)  # noqa: S101
    assert len(training_indices) + len(validation_indices) + len(test_indices) == len(all_data)  # noqa: S101
    assert (all_data.multi_view_data.iloc[training_indices + validation_indices].angle != "frontal").all()  # noqa: S101

    # Create the subsets
    train_data, val_data, test_data = (
        Subset(all_data, indices) for indices in (training_indices, validation_indices, test_indices)
    )

    # Create dataloaders
    train_dataloader = (
        DataLoader(train_data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
        if len(training_indices) > 0
        else None
    )
    validation_dataloader = (
        DataLoader(val_data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
        if len(validation_indices) > 0
        else None
    )
    test_dataloader = (
        DataLoader(test_data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
        if len(test_indices) > 0
        else None
    )

    return train_dataloader, validation_dataloader, test_dataloader

revert_model_image

revert_model_image(
    image: Tensor, add_mean: bool
) -> ndarray

Revert a model-input-image to its original form.

Source code in code/facesim3d/modeling/VGG/prepare_data.py

def revert_model_image(image: torch.Tensor, add_mean: bool) -> np.ndarray:
    """Revert a model-input-image to its original form."""
    if add_mean:
        image += torch.Tensor(np.array([129.1863, 104.7624, 93.5940])).to(image.dtype).view(1, 3, 1, 1)
    return image[0].permute(1, 2, 0).to("cpu").numpy().astype(np.uint8)