visualization.py

import pickle
from collections import Counter
from itertools import product

import matplotlib
import matplotlib.patches as patches
import numpy as np
import torchvision.transforms as transforms
from matplotlib import gridspec
from matplotlib import pyplot as plt
from matplotlib.patches import ConnectionPatch, ConnectionStyle
from PIL import Image

connectionstyle = ConnectionStyle("Arc3, rad=0.2")

display_transform = transforms.Compose(
    [transforms.Resize(240), transforms.CenterCrop((240, 240))]
)
display_transform_knn = transforms.Compose(
    [transforms.Resize(256), transforms.CenterCrop((224, 224))]
)


def keep_top_k(input_array, K=5):
    """
    return top 5 (k) from numpy array
    """
    top_5 = np.sort(input_array.reshape(-1))[::-1][K - 1]
    masked = np.zeros_like(input_array)
    masked[input_array >= top_5] = 1
    return masked


def arg_topK(inputarray, topK=5):
    """
    returns indicies related to top K element (largest)
    """
    return np.argsort(inputarray.T.reshape(-1))[::-1][:topK]


def plot_from_reranker_output(reranker_output, draw_box=True, draw_arcs=True):
    """
    visualize chm results from a reranker output dict
    """

    ### SET COLORS
    cmap = matplotlib.cm.get_cmap("gist_rainbow")
    rgba = cmap(0.5)
    colors = []
    for k in range(5):
        colors.append(cmap(k / 5.0))

    ### SET POINTS
    A = np.linspace(1 + 17, 240 - 17 - 1, 7)
    point_list = list(product(A, A))

    nrow = 4
    ncol = 7

    fig = plt.figure(figsize=(32, 18))
    gs = gridspec.GridSpec(
        nrow,
        ncol,
        width_ratios=[1, 0.2, 1, 1, 1, 1, 1],
        height_ratios=[1, 1, 1, 1],
        wspace=0.1,
        hspace=0.1,
        top=0.9,
        bottom=0.05,
        left=0.17,
        right=0.845,
    )
    axes = [[None for n in range(ncol - 1)] for x in range(nrow)]

    for i in range(4):
        axes[i] = []
        for j in range(7):
            if j != 1:
                if (i, j) in [(2, 0), (3, 0)]:
                    axes[i].append(new_ax)
                else:
                    new_ax = plt.subplot(gs[i, j])
                    new_ax.set_xticklabels([])
                    new_ax.set_xticks([])
                    new_ax.set_yticklabels([])
                    new_ax.set_yticks([])
                    new_ax.axis("off")
                    axes[i].append(new_ax)

    ##################### DRAW EVERYTHING
    axes[0][0].imshow(
        display_transform(Image.open(reranker_output["q"]).convert("RGB"))
    )
    axes[0][0].set_title(
        f'Query - K={reranker_output["K"]}, N={reranker_output["N"]}', fontsize=21
    )

    axes[1][0].imshow(
        display_transform(Image.open(reranker_output["q"]).convert("RGB"))
    )
    axes[1][0].set_title(f'Query - K={reranker_output["K"]}', fontsize=21)

    # axes[2][0].imshow(display_transform(Image.open(reranker_output['q'])))

    # CHM Top5
    for i in range(min(5, reranker_output["chm-prediction-confidence"])):
        axes[0][1 + i].imshow(
            display_transform(
                Image.open(reranker_output["chm-nearest-neighbors"][i]).convert("RGB")
            )
        )
        axes[0][1 + i].set_title(f"CHM-Corr - Top - {i+1}", fontsize=21)

    if reranker_output["chm-prediction-confidence"] < 5:
        for i in range(reranker_output["chm-prediction-confidence"], 5):
            axes[0][1 + i].imshow(Image.new(mode="RGB", size=(224, 224), color="white"))
            axes[0][1 + i].set_title(f"", fontsize=21)

    # KNN top5
    for i in range(min(5, reranker_output["knn-prediction-confidence"])):
        axes[1][1 + i].imshow(
            display_transform_knn(
                Image.open(reranker_output["knn-nearest-neighbors"][i]).convert("RGB")
            )
        )
        axes[1][1 + i].set_title(f"kNN - Top - {i+1}", fontsize=21)

    if reranker_output["knn-prediction-confidence"] < 5:
        for i in range(reranker_output["knn-prediction-confidence"], 5):
            axes[1][1 + i].imshow(Image.new(mode="RGB", size=(240, 240), color="white"))
            axes[1][1 + i].set_title(f"", fontsize=21)

    for i in range(min(5, reranker_output["chm-prediction-confidence"])):
        axes[2][i + 1].imshow(
            display_transform(Image.open(reranker_output["q"]).convert("RGB"))
        )

    # Lower ROWs CHM Top5
    for i in range(min(5, reranker_output["chm-prediction-confidence"])):
        axes[3][1 + i].imshow(
            display_transform(
                Image.open(reranker_output["chm-nearest-neighbors"][i]).convert("RGB")
            )
        )

    if reranker_output["chm-prediction-confidence"] < 5:
        for i in range(reranker_output["chm-prediction-confidence"], 5):
            axes[2][i + 1].imshow(Image.new(mode="RGB", size=(240, 240), color="white"))
            axes[3][1 + i].imshow(Image.new(mode="RGB", size=(240, 240), color="white"))

    nzm = reranker_output["non_zero_mask"]
    # Go throught top 5 nearest images

    # #################################################################################
    if draw_box:
        # SQUARAES
        for NC in range(min(5, reranker_output["chm-prediction-confidence"])):
            # ON SOURCE
            valid_patches_source = arg_topK(
                reranker_output["masked_cos_values"][NC], topK=nzm
            )

            # ON QUERY
            target_masked_patches = arg_topK(
                reranker_output["masked_cos_values"][NC], topK=nzm
            )
            valid_patches_target = [
                reranker_output["correspondance_map"][NC][x]
                for x in target_masked_patches
            ]
            valid_patches_target = [(x[0] * 7) + x[1] for x in valid_patches_target]

            patch_colors = [c for c in colors]
            overlaps = [
                item
                for item, count in Counter(valid_patches_target).items()
                if count > 1
            ]

            for O in overlaps:
                indices = [i for i, val in enumerate(valid_patches_target) if val == O]
                for ii in indices[1:]:
                    patch_colors[ii] = patch_colors[indices[0]]

            for i in valid_patches_source:
                Psource = point_list[i]
                rect = patches.Rectangle(
                    (Psource[0] - 16, Psource[1] - 16),
                    32,
                    32,
                    linewidth=2,
                    edgecolor=patch_colors[valid_patches_source.tolist().index(i)],
                    facecolor="none",
                    alpha=1,
                )
                axes[2][1 + NC].add_patch(rect)

            for i in valid_patches_target:
                Psource = point_list[i]
                rect = patches.Rectangle(
                    (Psource[0] - 16, Psource[1] - 16),
                    32,
                    32,
                    linewidth=2,
                    edgecolor=patch_colors[valid_patches_target.index(i)],
                    facecolor="none",
                    alpha=1,
                )
                axes[3][1 + NC].add_patch(rect)

    #################################################################################
    # Show correspondence lines and points
    if draw_arcs:
        for CK in range(min(5, reranker_output["chm-prediction-confidence"])):
            target_keypoints = []
            topk_index = arg_topK(reranker_output["masked_cos_values"][CK], topK=nzm)
            for i in range(nzm):  # Number of Connections
                con = ConnectionPatch(
                    xyA=(
                        reranker_output["src-keypoints"][CK][i, 0],
                        reranker_output["src-keypoints"][CK][i, 1],
                    ),
                    xyB=(
                        reranker_output["tgt-keypoints"][CK][i, 0],
                        reranker_output["tgt-keypoints"][CK][i, 1],
                    ),
                    coordsA="data",
                    coordsB="data",
                    axesA=axes[2][1 + CK],
                    axesB=axes[3][1 + CK],
                    color=colors[i],
                    connectionstyle=connectionstyle,
                    shrinkA=1.0,
                    shrinkB=1.0,
                    linewidth=1,
                )

                axes[3][1 + CK].add_artist(con)

            # Scatter Plot
            axes[2][1 + CK].scatter(
                reranker_output["src-keypoints"][CK][:, 0],
                reranker_output["src-keypoints"][CK][:, 1],
                c=colors[:nzm],
                s=10,
            )
            axes[3][1 + CK].scatter(
                reranker_output["tgt-keypoints"][CK][:, 0],
                reranker_output["tgt-keypoints"][CK][:, 1],
                c=colors[:nzm],
                s=10,
            )

    fig.text(
        0.5,
        0.95,
        f"CHM-Corr Prediction: {reranker_output['chm-prediction']}",
        ha="center",
        va="bottom",
        color="black",
        fontsize=22,
    )

    return fig


def plot_from_reranker_corrmap(reranker_output, draw_box=True):
    """
    visualize chm results from a reranker output dict
    """

    ### SET COLORS
    cmap = matplotlib.cm.get_cmap("gist_rainbow")
    rgba = cmap(0.5)
    colors = []
    for k in range(5):
        colors.append(cmap(k / 5.0))

    ### SET POINTS
    A = np.linspace(1 + 17, 240 - 17 - 1, 7)
    point_list = list(product(A, A))

    fig, axes = plt.subplots(
        2,
        7,
        figsize=(25, 8),
        gridspec_kw={
            "wspace": 0,
            "hspace": 0,
            "width_ratios": [1, 0.28, 1, 1, 1, 1, 1],
        },
        facecolor=(1, 1, 1),
    )

    for i in range(2):
        for j in range(7):
            axes[i][j].axis("off")

    axes[0][0].imshow(
        display_transform(Image.open(reranker_output["q"]).convert("RGB"))
    )

    for i in range(min(5, reranker_output["chm-prediction-confidence"])):
        axes[0][2 + i].imshow(
            display_transform(Image.open(reranker_output["q"]).convert("RGB"))
        )

    # Lower ROWs CHM Top5
    for i in range(min(5, reranker_output["chm-prediction-confidence"])):
        axes[1][2 + i].imshow(
            display_transform(
                Image.open(reranker_output["chm-nearest-neighbors"][i]).convert("RGB")
            )
        )

    if reranker_output["chm-prediction-confidence"] < 5:
        for i in range(reranker_output["chm-prediction-confidence"], 5):
            axes[0][2 + i].imshow(Image.new(mode="RGB", size=(240, 240), color="white"))
            axes[1][2 + i].imshow(Image.new(mode="RGB", size=(240, 240), color="white"))

    nzm = reranker_output["non_zero_mask"]
    # Go throught top 5 nearest images

    # #################################################################################
    if draw_box:
        # SQUARAES
        for NC in range(min(5, reranker_output["chm-prediction-confidence"])):
            # ON SOURCE
            valid_patches_source = arg_topK(
                reranker_output["masked_cos_values"][NC], topK=nzm
            )

            # ON QUERY
            target_masked_patches = arg_topK(
                reranker_output["masked_cos_values"][NC], topK=nzm
            )
            valid_patches_target = [
                reranker_output["correspondance_map"][NC][x]
                for x in target_masked_patches
            ]
            valid_patches_target = [(x[0] * 7) + x[1] for x in valid_patches_target]

            patch_colors = [c for c in colors]
            overlaps = [
                item
                for item, count in Counter(valid_patches_target).items()
                if count > 1
            ]

            for O in overlaps:
                indices = [i for i, val in enumerate(valid_patches_target) if val == O]
                for ii in indices[1:]:
                    patch_colors[ii] = patch_colors[indices[0]]

            for i in valid_patches_source:
                Psource = point_list[i]
                rect = patches.Rectangle(
                    (Psource[0] - 16, Psource[1] - 16),
                    32,
                    32,
                    linewidth=2,
                    edgecolor=patch_colors[valid_patches_source.tolist().index(i)],
                    facecolor="none",
                    alpha=1,
                )
                axes[0][2 + NC].add_patch(rect)

            for i in valid_patches_target:
                Psource = point_list[i]
                rect = patches.Rectangle(
                    (Psource[0] - 16, Psource[1] - 16),
                    32,
                    32,
                    linewidth=2,
                    edgecolor=patch_colors[valid_patches_target.index(i)],
                    facecolor="none",
                    alpha=1,
                )
                axes[1][2 + NC].add_patch(rect)

    return fig, reranker_output["chm-prediction"]