netdissect/actviz.py

import os
import numpy
from scipy.interpolate import RectBivariateSpline

def activation_visualization(image, data, level, alpha=0.5, source_shape=None,
        crop=False, zoom=None, border=2, negate=False, return_mask=False,
        **kwargs):
    """
    Makes a visualiztion image of activation data overlaid on the image.
    Params:
        image The original image.
        data The single channel feature map.
        alpha The darkening to apply in inactive regions of the image.
        level The threshold of activation levels to highlight.
    """
    if len(image.shape) == 2:
        # Puff up grayscale image to RGB.
        image = image[:,:,None] * numpy.array([[[1, 1, 1]]])
    surface = activation_surface(data, target_shape=image.shape[:2],
            source_shape=source_shape, **kwargs)
    if negate:
        surface = -surface
        level = -level
    if crop:
        # crop to source_shape
        if source_shape is not None:
            ch, cw = ((t - s) // 2 for s, t in zip(
                source_shape, image.shape[:2]))
            image = image[ch:ch+source_shape[0], cw:cw+source_shape[1]]
            surface = surface[ch:ch+source_shape[0], cw:cw+source_shape[1]]
        if crop is True:
            crop = surface.shape
        elif not hasattr(crop, '__len__'):
            crop = (crop, crop)
        if zoom is not None:
            source_rect = best_sub_rect(surface >= level, crop, zoom,
                    pad=border)
        else:
            source_rect = (0, surface.shape[0], 0, surface.shape[1])
        image = zoom_image(image, source_rect, crop)
        surface = zoom_image(surface, source_rect, crop)
    mask = (surface >= level)
    # Add a yellow border at the edge of the mask for contrast
    result = (mask[:, :, None] * (1 - alpha) + alpha) * image
    if border:
        edge = mask_border(mask)[:,:,None]
        result = numpy.maximum(edge * numpy.array([[[200, 200, 0]]]), result)
    if not return_mask:
        return result
    mask_image = (1 - mask[:, :, None]) * numpy.array(
            [[[0, 0, 0, 255 * (1 - alpha)]]], dtype=numpy.uint8)
    if border:
        mask_image = numpy.maximum(edge * numpy.array([[[200, 200, 0, 255]]]),
                mask_image)
    return result, mask_image

def activation_surface(data, target_shape=None, source_shape=None,
        scale_offset=None, deg=1, pad=True):
    """
    Generates an upsampled activation sample.
    Params:
        target_shape Shape of the output array.
        source_shape The centered shape of the output to match with data
            when upscaling.  Defaults to the whole target_shape.
        scale_offset The amount by which to scale, then offset data
            dimensions to end up with target dimensions.  A pair of pairs.
        deg Degree of interpolation to apply (1 = linear, etc).
        pad True to zero-pad the edge instead of doing a funny edge interp.
    """
    # Default is that nothing is resized.
    if target_shape is None:
        target_shape = data.shape
    # Make a default scale_offset to fill the image if there isn't one
    if scale_offset is None:
        scale = tuple(float(ts) / ds
                for ts, ds in zip(target_shape, data.shape))
        offset = tuple(0.5 * s - 0.5 for s in scale)
    else:
        scale, offset = (v for v in zip(*scale_offset))
    # Now we adjust offsets to take into account cropping and so on
    if source_shape is not None:
        offset = tuple(o + (ts - ss) / 2.0
                for o, ss, ts in zip(offset, source_shape, target_shape))
    # Pad the edge with zeros for sensible edge behavior
    if pad:
        zeropad = numpy.zeros(
                (data.shape[0] + 2, data.shape[1] + 2), dtype=data.dtype)
        zeropad[1:-1, 1:-1] = data
        data = zeropad
        offset = tuple((o - s) for o, s in zip(offset, scale))
    # Upsample linearly
    ty, tx = (numpy.arange(ts) for ts in target_shape)
    sy, sx = (numpy.arange(ss) * s + o
            for ss, s, o in zip(data.shape, scale, offset))
    levels = RectBivariateSpline(
            sy, sx, data, kx=deg, ky=deg)(ty, tx, grid=True)
    # Return the mask.
    return levels

def mask_border(mask, border=2):
    """Given a mask computes a border mask"""
    from scipy import ndimage
    struct = ndimage.generate_binary_structure(2, 2)
    erosion = numpy.ones((mask.shape[0] + 10, mask.shape[1] + 10), dtype='int')
    erosion[5:5+mask.shape[0], 5:5+mask.shape[1]] = ~mask
    for _ in range(border):
        erosion = ndimage.binary_erosion(erosion, struct)
    return ~mask ^ erosion[5:5+mask.shape[0], 5:5+mask.shape[1]]

def bounding_rect(mask, pad=0):
    """Returns (r, b, l, r) boundaries so that all nonzero pixels in mask
    have locations (i, j) with  t <= i < b, and l <= j < r."""
    nz = mask.nonzero()
    if len(nz[0]) == 0:
        # print('no pixels')
        return (0, mask.shape[0], 0, mask.shape[1])
    (t, b), (l, r) = [(max(0, p.min() - pad), min(s, p.max() + 1 + pad))
            for p, s in zip(nz, mask.shape)]
    return (t, b, l, r)

def best_sub_rect(mask, shape, max_zoom=None, pad=2):
    """Finds the smallest subrectangle containing all the nonzeros of mask,
    matching the aspect ratio of shape, and where the zoom-up ratio is no
    more than max_zoom"""
    t, b, l, r = bounding_rect(mask, pad=pad)
    height = max(b - t, int(round(float(shape[0]) * (r - l) / shape[1])))
    if max_zoom is not None:
        height = int(max(round(float(shape[0]) / max_zoom), height))
    width = int(round(float(shape[1]) * height / shape[0]))
    nt = min(mask.shape[0] - height, max(0, (b + t - height) // 2))
    nb = nt + height
    nl = min(mask.shape[1] - width, max(0, (r + l - width) // 2))
    nr = nl + width
    return (nt, nb, nl, nr)

def zoom_image(img, source_rect, target_shape=None):
    """Zooms pixels from the source_rect of img to target_shape."""
    import warnings
    from scipy.ndimage import zoom
    if target_shape is None:
        target_shape = img.shape
    st, sb, sl, sr = source_rect
    source = img[st:sb, sl:sr]
    if source.shape == target_shape:
        return source
    zoom_tuple = tuple(float(t) / s
            for t, s in zip(target_shape, source.shape[:2])
            ) + (1,) * (img.ndim - 2)
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', UserWarning) # "output shape of zoom"
        target = zoom(source, zoom_tuple)
    assert target.shape[:2] == target_shape, (target.shape, target_shape)
    return target

def scale_offset(dilations):
    if len(dilations) == 0:
        return (1, 0)
    scale, offset = scale_offset(dilations[1:])
    kernel, stride, padding = dilations[0]
    scale *= stride
    offset *= stride
    offset += (kernel - 1) / 2.0 - padding
    return scale, offset

def choose_level(feature_map, percentile=0.8):
    '''
    Chooses the top 80% level (or whatever the level chosen).
    '''
    data_range = numpy.sort(feature_map.flatten())
    return numpy.interp(
            percentile, numpy.linspace(0, 1, len(data_range)), data_range)

def dilations(modulelist):
    result = []
    for module in modulelist:
        settings = tuple(getattr(module, n, d)
            for n, d in (('kernel_size', 1), ('stride', 1), ('padding', 0)))
        settings = (((s, s) if not isinstance(s, tuple) else s)
            for s in settings)
        if settings != ((1, 1), (1, 1), (0, 0)):
            result.append(zip(*settings))
    return zip(*result)

def grid_scale_offset(modulelist):
    '''Returns (yscale, yoffset), (xscale, xoffset) given a list of modules'''
    return tuple(scale_offset(d) for d in dilations(modulelist))