utils/visualization.py

import os
import io
import torch
import numpy as np
from termcolor import colored
try:
    # os.environ["PYOPENGL_PLATFORM"] = "osmesa"
    os.environ["PYOPENGL_PLATFORM"] = "egl"
    import pyrender
except:
    print(colored('pyrender is not correctly imported.', 'red'))
import matplotlib
from matplotlib import colormaps
from matplotlib.colors import LightSource
import matplotlib.pyplot as plt
import math
import cv2
import trimesh
from sklearn.decomposition import PCA
from scipy.spatial.transform import Rotation as R
import torchvision
from .transforms import adjust_colors

BASE_COLORS = np.loadtxt(os.path.abspath(os.path.join(__file__, "../colors.txt")), skiprows=0)/255.
BASE_COLORS = adjust_colors(BASE_COLORS,
                            saturation_threshold = 0.3, 
                            brightness_threshold = 0.8)


def get_colors_rgb(size):
    # np.random.seed(131)
    return BASE_COLORS[np.random.choice(BASE_COLORS.shape[0], size=size, replace=False)]

def tensor_to_BGR(img_tensor):
    img = img_tensor.numpy()*255
    img = img.astype(np.uint8).transpose((1,2,0))[:,:,::-1].copy()
    return img

def pad_img(img, pad_size = None, pad_color_offset = 127):
    if not isinstance(img, np.ndarray):
        img = tensor_to_BGR(img.detach().cpu())
    if pad_size is None:
        pad_size = max(img.shape[0],img.shape[1])

    pad = np.zeros((pad_size,pad_size,img.shape[-1]), dtype=img.dtype) + pad_color_offset
    pad[:img.shape[0], :img.shape[1]] = img.copy()
    return pad


def vis_scale_img(img, scale_map, conf_thresh = 0.3, patch_size=14):
    cmap = plt.get_cmap('coolwarm')

    vis_map = np.zeros((scale_map.shape[0]*patch_size, scale_map.shape[1]*patch_size, 3), dtype=np.uint8)
    loc_i, loc_j = torch.where(scale_map[:,:,0] > conf_thresh)
    for (i, j) in zip(loc_i, loc_j):
        scale = round(math.sqrt(scale_map[i,j,1].item()),2)
        vis_map[i*patch_size: (i+1)*patch_size, j*patch_size: (j+1)*patch_size] = (np.array(cmap(1-scale)[:3][::-1])*255).astype(np.uint8)
    vis_map = pad_img(vis_map, pad_color_offset=0)
    img = pad_img(img)
    # print(img.shape, vis_map.shape)
    assert img.shape == vis_map.shape

    white_img = 0.6*img + 0.4*np.array((255,255,255))

    valid_mask = (vis_map > 0)
    visible_weight = 0.8
    img = vis_map * valid_mask * visible_weight +\
                img * valid_mask * (1-visible_weight)+\
                white_img * (1-valid_mask)

    # draw patches
    loc_i, loc_j = torch.where(scale_map[:,:,0]+1)
    for (i, j) in zip(loc_i.tolist(), loc_j.tolist()):
        cv2.rectangle(img, (j*patch_size, i*patch_size), ((j+1)*patch_size, (i+1)*patch_size),
                        color=(255,255,255), thickness = 2 )

    return img

def vis_meshes_img(img, verts, smpl_faces, cam_intrinsics, colors = None, padding = True):
    if not isinstance(img, np.ndarray):
        img = tensor_to_BGR(img.detach().cpu())

    if padding:
        pad_size = max(img.shape[0],img.shape[1])
        img = pad_img(img, pad_size)

    if colors is not None:
        assert len(colors) == len(verts)

    if len(cam_intrinsics.flatten()) == 9:
        cam_intrinsics = cam_intrinsics.reshape(3,3)
        rgb, depth = render_mesh(img.shape[0],img.shape[1],verts,smpl_faces,cam_intrinsics,colors)
        valid_mask = (depth > 0)[:,:,None] 
        visible_weight = 1.
        rendered_img = rgb[:,:,::-1] * valid_mask * visible_weight +\
                        img * valid_mask * (1-visible_weight)+\
                        img * (1-valid_mask)
    else:
        rendered_img = img
        for i, cam_int in enumerate(cam_intrinsics):
            rgb, depth = render_mesh(img.shape[0],img.shape[1],[verts[i]],smpl_faces,cam_int,colors)
            valid_mask = (depth > 0)[:,:,None] 
            visible_weight = 0.8
            rendered_img = rgb[:,:,::-1] * valid_mask * visible_weight +\
                            rendered_img * valid_mask * (1-visible_weight)+\
                            rendered_img * (1-valid_mask)
    rendered_img = rendered_img.astype(np.uint8)

    return rendered_img


def vis_joints_img(img, j2ds):
    pass

def vis_sat(img, input_size, patch_size, sat_dict, bid, padding=True):
    if not isinstance(img, np.ndarray):
        img = tensor_to_BGR(img.detach().cpu())

    assert max(img.shape[0], img.shape[1]) == input_size
    if padding:
        img = pad_img(img, input_size)

    # visualize patches
    pos_y, pos_x = sat_dict['pos_y'][bid], sat_dict['pos_x'][bid]
    pos_y = (pos_y * input_size).detach().int().cpu().numpy()
    pos_x = (pos_x * input_size).detach().int().cpu().numpy()

    lvls = sat_dict['lvl']
    if lvls is None:
        lvl = np.zeros(len(pos_x),dtype=int)
    else:
        lvl = lvls[bid].detach().int().cpu().numpy()

    for (cx, cy, l) in zip(pos_x, pos_y, lvl):
        if l == 0:
            half_patch = patch_size//2
            # color = (139, 97, 233)
            color = (173,178,241)
        elif l == 1:
            half_patch = patch_size
            # color = (246, 222, 118)
            color = (239,198,175)
        elif l >= 2:
            half_patch = patch_size*(2**(l-1))
            # color = (0,0,0)
            color = (255, 255, 255)
        else:
            raise NotImplementedError

        x1, x2 = cx - half_patch, cx + half_patch
        y1, y2 = cy - half_patch, cy + half_patch

        if l>0:
            k = 7*l
            img[y1:y2,x1:x2] = 0.5*cv2.blur(img[y1:y2,x1:x2].copy(),(k,k)) + 0.5*np.array(color)
        else:
            # pass
            img[y1:y2,x1:x2] = 0.5*img[y1:y2,x1:x2].copy() + 0.5*np.array(color)


    for (cx, cy, l) in zip(pos_x, pos_y, lvl):
        if l == 0:
            half_patch = patch_size//2
            color = (139, 97, 233)
        elif l == 1:
            half_patch = patch_size
            color = (246, 222, 118)
        elif l >= 2:
            half_patch = patch_size*(2**(l-1))
            color = (255, 255, 255)
        else:
            raise NotImplementedError
        
        x1, x2 = cx - half_patch, cx + half_patch
        y1, y2 = cy - half_patch, cy + half_patch

        cv2.rectangle(img, (x1, y1), (x2, y2),
            color=(255,255,255), thickness = 2 )
        

    return img

def vis_boxes(img, boxes, padding=True, color = (0,0,255)):
    if not isinstance(img, np.ndarray):
        img = tensor_to_BGR(img.detach().cpu())
    if padding:
        pad_size = max(img.shape[0],img.shape[1])
        img = pad_img(img, pad_size)
    
    for bbox in boxes:
        bbox = bbox.int().tolist()
        cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
            color=color, thickness = 2 )
    
    return img


def get_img_from_fig(fig, dpi=120):
    buf = io.BytesIO()
    fig.savefig(buf, format="png", dpi=dpi, transparent=False, bbox_inches="tight", pad_inches=0)
    buf.seek(0)
    img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)
    buf.close()
    img = cv2.imdecode(img_arr, 1)
    # img = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)
    return img


def render_mesh(height, width, meshes, face, cam_intrinsics, colors = None):
    
    # renderer
    scene = pyrender.Scene(ambient_light=(0.3, 0.3, 0.3))
    renderer = pyrender.OffscreenRenderer(viewport_width=width, viewport_height=height, point_size=1.0)

    # light
    light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=0.8)
    light_pose = np.eye(4)
    light_pose[:3, 3] = np.array([0, -1, 1])
    scene.add(light, pose=light_pose)
    light_pose[:3, 3] = np.array([0, 1, 1])
    scene.add(light, pose=light_pose)
    light_pose[:3, 3] = np.array([1, 1, 2])
    scene.add(light, pose=light_pose)

    # mesh
    if colors is None:
        colors = get_colors_rgb(len(meshes))

    for i, mesh in enumerate(meshes):
        mesh = trimesh.Trimesh(mesh, face)
        rot = trimesh.transformations.rotation_matrix(np.radians(180), [1, 0, 0])
        mesh.apply_transform(rot)
        material = pyrender.MetallicRoughnessMaterial(metallicFactor=0.0, alphaMode='OPAQUE', baseColorFactor=(*colors[i], 1.0))
        mesh = pyrender.Mesh.from_trimesh(mesh, material=material, smooth=True)

        scene.add(mesh, f'mesh_{i}')


    # camera
    f=np.array([cam_intrinsics[0,0],cam_intrinsics[1,1]])
    c=cam_intrinsics[0:2,2]
    camera = pyrender.camera.IntrinsicsCamera(fx=f[0], fy=f[1], cx=c[0], cy=c[1])
    scene.add(camera)

    # render
    rgb, depth = renderer.render(scene, flags=pyrender.RenderFlags.RGBA)
    rgb = rgb[:,:,:3].astype(np.float32)
    renderer.delete()
    return rgb, depth