Upload 176 files

app.py

@@ -0,0 +1,178 @@
+# Copyright (C) 2020 * Ltd. All rights reserved.
+# author : Sanghyeon Jo <[email protected]>
+
+import gradio as gr
+
+import os
+import sys
+import copy
+import shutil
+import random
+import argparse
+import numpy as np
+
+import imageio
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torchvision import transforms
+from torch.utils.tensorboard import SummaryWriter
+
+from torch.utils.data import DataLoader
+
+from core.puzzle_utils import *
+from core.networks import *
+from core.datasets import *
+
+from tools.general.io_utils import *
+from tools.general.time_utils import *
+from tools.general.json_utils import *
+
+from tools.ai.log_utils import *
+from tools.ai.demo_utils import *
+from tools.ai.optim_utils import *
+from tools.ai.torch_utils import *
+from tools.ai.evaluate_utils import *
+
+from tools.ai.augment_utils import *
+from tools.ai.randaugment import *
+
+parser = argparse.ArgumentParser()
+
+###############################################################################
+# Dataset
+###############################################################################
+parser.add_argument('--seed', default=2606, type=int)
+parser.add_argument('--num_workers', default=4, type=int)
+parser.add_argument('--data_dir', default='../VOCtrainval_11-May-2012/', type=str)
+
+###############################################################################
+# Network
+###############################################################################
+parser.add_argument('--architecture', default='DeepLabv3+', type=str)
+parser.add_argument('--backbone', default='resnet50', type=str)
+parser.add_argument('--mode', default='fix', type=str)
+parser.add_argument('--use_gn', default=True, type=str2bool)
+
+###############################################################################
+# Inference parameters
+###############################################################################
+parser.add_argument('--tag', default='', type=str)
+
+parser.add_argument('--domain', default='val', type=str)
+
+parser.add_argument('--scales', default='0.5,1.0,1.5,2.0', type=str)
+parser.add_argument('--iteration', default=10, type=int)
+
+if __name__ == '__main__':
+    ###################################################################################
+    # Arguments
+    ###################################################################################
+    args = parser.parse_args()
+
+    model_dir = create_directory('./experiments/models/')
+    model_path = model_dir + f'DeepLabv3+@ResNeSt-101@[email protected]'
+
+    if 'train' in args.domain:
+        args.tag += '@train'
+    else:
+        args.tag += '@' + args.domain
+
+    args.tag += '@scale=%s' % args.scales
+    args.tag += '@iteration=%d' % args.iteration
+
+    set_seed(args.seed)
+    log_func = lambda string='': print(string)
+
+    ###################################################################################
+    # Transform, Dataset, DataLoader
+    ###################################################################################
+    imagenet_mean = [0.485, 0.456, 0.406]
+    imagenet_std = [0.229, 0.224, 0.225]
+
+    normalize_fn = Normalize(imagenet_mean, imagenet_std)
+
+    # for mIoU
+    meta_dic = read_json('./data/VOC_2012.json')
+
+    ###################################################################################
+    # Network
+    ###################################################################################
+    if args.architecture == 'DeepLabv3+':
+        model = DeepLabv3_Plus(args.backbone, num_classes=meta_dic['classes'] + 1, mode=args.mode,
+                               use_group_norm=args.use_gn)
+    elif args.architecture == 'Seg_Model':
+        model = Seg_Model(args.backbone, num_classes=meta_dic['classes'] + 1)
+    elif args.architecture == 'CSeg_Model':
+        model = CSeg_Model(args.backbone, num_classes=meta_dic['classes'] + 1)
+
+    model = model.cuda()
+    model.eval()
+
+    log_func('[i] Architecture is {}'.format(args.architecture))
+    log_func('[i] Total Params: %.2fM' % (calculate_parameters(model)))
+    log_func()
+
+    load_model(model, model_path, parallel=False)
+
+    #################################################################################################
+    # Evaluation
+    #################################################################################################
+    eval_timer = Timer()
+    scales = [float(scale) for scale in args.scales.split(',')]
+
+    model.eval()
+    eval_timer.tik()
+
+
+    def inference(images, image_size):
+        images = images.cuda()
+
+        logits = model(images)
+        logits = resize_for_tensors(logits, image_size)
+
+        logits = logits[0] + logits[1].flip(-1)
+        logits = get_numpy_from_tensor(logits).transpose((1, 2, 0))
+        return logits
+
+
+    def predict_image(ori_image):
+        with torch.no_grad():
+            ori_w, ori_h = ori_image.size
+
+            cams_list = []
+
+            for scale in scales:
+                image = copy.deepcopy(ori_image)
+                image = image.resize((round(ori_w * scale), round(ori_h * scale)), resample=PIL.Image.BICUBIC)
+
+                image = normalize_fn(image)
+                image = image.transpose((2, 0, 1))
+
+                image = torch.from_numpy(image)
+                flipped_image = image.flip(-1)
+
+                images = torch.stack([image, flipped_image])
+
+                cams = inference(images, (ori_h, ori_w))
+                cams_list.append(cams)
+
+            preds = np.sum(cams_list, axis=0)
+            preds = F.softmax(torch.from_numpy(preds), dim=-1).numpy()
+
+            if args.iteration > 0:
+                preds = crf_inference(np.asarray(ori_image), preds.transpose((2, 0, 1)), t=args.iteration)
+                pred_mask = np.argmax(preds, axis=0)
+            else:
+                pred_mask = np.argmax(preds, axis=-1)
+
+            return pred_mask.astype(np.uint8)
+
+
+    demo = gr.Interface(
+        fn=predict_image,
+        inputs="image",
+        outputs="image"
+    )

core/abc_modules.py

@@ -0,0 +1,50 @@
+
+import math
+
+import torch
+import torch.nn as nn
+
+from abc import ABC
+
+class ABC_Model(ABC):
+    def global_average_pooling_2d(self, x, keepdims=False):
+        x = torch.mean(x.view(x.size(0), x.size(1), -1), -1)
+        if keepdims:
+            x = x.view(x.size(0), x.size(1), 1, 1)
+        return x
+    
+    def initialize(self, modules):
+        for m in modules:
+            if isinstance(m, nn.Conv2d):
+                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                # m.weight.data.normal_(0, math.sqrt(2. / n))
+                torch.nn.init.kaiming_normal_(m.weight)
+                
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+    
+    def get_parameter_groups(self, print_fn=print):
+        groups = ([], [], [], [])
+
+        for name, value in self.named_parameters():
+            # pretrained weights
+            if 'model' in name:
+                if 'weight' in name:
+                    # print_fn(f'pretrained weights : {name}')
+                    groups[0].append(value)
+                else:
+                    # print_fn(f'pretrained bias : {name}')
+                    groups[1].append(value)
+                    
+            # scracthed weights
+            else:
+                if 'weight' in name:
+                    if print_fn is not None:
+                        print_fn(f'scratched weights : {name}')
+                    groups[2].append(value)
+                else:
+                    if print_fn is not None:
+                        print_fn(f'scratched bias : {name}')
+                    groups[3].append(value)
+        return groups

core/aff_utils.py

@@ -0,0 +1,178 @@
+import torch
+import torch.nn.functional as F
+import numpy as np
+
+class PathIndex:
+    def __init__(self, radius, default_size):
+        self.radius = radius
+        self.radius_floor = int(np.ceil(radius) - 1)
+
+        self.search_paths, self.search_dst = self.get_search_paths_dst(self.radius)
+        self.path_indices, self.src_indices, self.dst_indices = self.get_path_indices(default_size)
+
+    def get_search_paths_dst(self, max_radius=5):
+        coord_indices_by_length = [[] for _ in range(max_radius * 4)]
+
+        search_dirs = []
+        for x in range(1, max_radius):
+            search_dirs.append((0, x))
+
+        for y in range(1, max_radius):
+            for x in range(-max_radius + 1, max_radius):
+                if x * x + y * y < max_radius ** 2:
+                    search_dirs.append((y, x))
+
+        for dir in search_dirs:
+            length_sq = dir[0] ** 2 + dir[1] ** 2
+            path_coords = []
+
+            min_y, max_y = sorted((0, dir[0]))
+            min_x, max_x = sorted((0, dir[1]))
+
+            for y in range(min_y, max_y + 1):
+                for x in range(min_x, max_x + 1):
+
+                    dist_sq = (dir[0] * x - dir[1] * y) ** 2 / length_sq
+
+                    if dist_sq < 1:
+                        path_coords.append([y, x])
+
+            path_coords.sort(key=lambda x: -abs(x[0]) - abs(x[1]))
+            path_length = len(path_coords)
+
+            coord_indices_by_length[path_length].append(path_coords)
+
+        path_list_by_length = [np.asarray(v) for v in coord_indices_by_length if v]
+        path_destinations = np.concatenate([p[:, 0] for p in path_list_by_length], axis=0)
+
+        return path_list_by_length, path_destinations
+
+    def get_path_indices(self, size):
+        full_indices = np.reshape(np.arange(0, size[0] * size[1], dtype=np.int64), (size[0], size[1]))
+
+        cropped_height = size[0] - self.radius_floor
+        cropped_width = size[1] - 2 * self.radius_floor
+
+        path_indices = []
+        for paths in self.search_paths:
+
+            path_indices_list = []
+            for p in paths:
+                coord_indices_list = []
+
+                for dy, dx in p:
+                    coord_indices = full_indices[dy:dy + cropped_height,
+                                    self.radius_floor + dx:self.radius_floor + dx + cropped_width]
+                    coord_indices = np.reshape(coord_indices, [-1])
+
+                    coord_indices_list.append(coord_indices)
+
+                path_indices_list.append(coord_indices_list)
+
+            path_indices.append(np.array(path_indices_list))
+
+        src_indices = np.reshape(full_indices[:cropped_height, self.radius_floor:self.radius_floor + cropped_width], -1)
+        dst_indices = np.concatenate([p[:,0] for p in path_indices], axis=0)
+
+        return path_indices, src_indices, dst_indices
+
+
+def edge_to_affinity(edge, paths_indices):
+    aff_list = []
+    edge = edge.view(edge.size(0), -1)
+
+    for i in range(len(paths_indices)):
+        if isinstance(paths_indices[i], np.ndarray):
+            paths_indices[i] = torch.from_numpy(paths_indices[i])
+        paths_indices[i] = paths_indices[i].cuda(non_blocking=True)
+
+    for ind in paths_indices:
+        ind_flat = ind.view(-1)
+        dist = torch.index_select(edge, dim=-1, index=ind_flat)
+        dist = dist.view(dist.size(0), ind.size(0), ind.size(1), ind.size(2))
+        aff = torch.squeeze(1 - F.max_pool2d(dist, (dist.size(2), 1)), dim=2)
+        aff_list.append(aff)
+    aff_cat = torch.cat(aff_list, dim=1)
+
+    return aff_cat
+
+
+def affinity_sparse2dense(affinity_sparse, ind_from, ind_to, n_vertices):
+    ind_from = torch.from_numpy(ind_from)
+    ind_to = torch.from_numpy(ind_to)
+
+    affinity_sparse = affinity_sparse.view(-1).cpu()
+    ind_from = ind_from.repeat(ind_to.size(0)).view(-1)
+    ind_to = ind_to.view(-1)
+
+    indices = torch.stack([ind_from, ind_to])
+    indices_tp = torch.stack([ind_to, ind_from])
+
+    indices_id = torch.stack([torch.arange(0, n_vertices).long(), torch.arange(0, n_vertices).long()])
+
+    affinity_dense = torch.sparse.FloatTensor(torch.cat([indices, indices_id, indices_tp], dim=1),
+                                       torch.cat([affinity_sparse, torch.ones([n_vertices]), affinity_sparse])).to_dense().cuda()
+
+    return affinity_dense
+
+
+def to_transition_matrix(affinity_dense, beta, times):
+    scaled_affinity = torch.pow(affinity_dense, beta)
+
+    trans_mat = scaled_affinity / torch.sum(scaled_affinity, dim=0, keepdim=True)
+    for _ in range(times):
+        trans_mat = torch.matmul(trans_mat, trans_mat)
+
+    return trans_mat
+
+def propagate_to_edge(x, edge, radius=5, beta=10, exp_times=8):
+    height, width = x.shape[-2:]
+
+    hor_padded = width+radius*2
+    ver_padded = height+radius
+
+    path_index = PathIndex(radius=radius, default_size=(ver_padded, hor_padded))
+    
+    edge_padded = F.pad(edge, (radius, radius, 0, radius), mode='constant', value=1.0)
+    sparse_aff = edge_to_affinity(torch.unsqueeze(edge_padded, 0),
+                                  path_index.path_indices)
+
+    dense_aff = affinity_sparse2dense(sparse_aff, path_index.src_indices,
+                                      path_index.dst_indices, ver_padded * hor_padded)
+    dense_aff = dense_aff.view(ver_padded, hor_padded, ver_padded, hor_padded)
+    dense_aff = dense_aff[:-radius, radius:-radius, :-radius, radius:-radius]
+    dense_aff = dense_aff.reshape(height * width, height * width)
+
+    trans_mat = to_transition_matrix(dense_aff, beta=beta, times=exp_times)
+
+    x = x.view(-1, height, width) * (1 - edge)
+
+    rw = torch.matmul(x.view(-1, height * width), trans_mat)
+    rw = rw.view(rw.size(0), 1, height, width)
+
+    return rw
+
+class GetAffinityLabelFromIndices():
+    def __init__(self, indices_from, indices_to):
+        self.indices_from = indices_from
+        self.indices_to = indices_to
+
+    def __call__(self, segm_map):
+        segm_map_flat = np.reshape(segm_map, -1)
+        
+        segm_label_from = np.expand_dims(segm_map_flat[self.indices_from], axis=0)
+        segm_label_to = segm_map_flat[self.indices_to]
+
+        valid_label = np.logical_and(np.less(segm_label_from, 21), np.less(segm_label_to, 21))
+
+        equal_label = np.equal(segm_label_from, segm_label_to)
+
+        pos_affinity_label = np.logical_and(equal_label, valid_label)
+
+        bg_pos_affinity_label = np.logical_and(pos_affinity_label, np.equal(segm_label_from, 0)).astype(np.float32)
+        fg_pos_affinity_label = np.logical_and(pos_affinity_label, np.greater(segm_label_from, 0)).astype(np.float32)
+
+        neg_affinity_label = np.logical_and(np.logical_not(equal_label), valid_label).astype(np.float32)
+
+        return torch.from_numpy(bg_pos_affinity_label), torch.from_numpy(fg_pos_affinity_label), torch.from_numpy(neg_affinity_label)
+

core/arch_resnest/resnest.py

@@ -0,0 +1,71 @@
+##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+## Created by: Hang Zhang
+## Email: [email protected]
+## Copyright (c) 2020
+##
+## LICENSE file in the root directory of this source tree 
+##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+"""ResNeSt models"""
+
+import torch
+from .resnet import ResNet, Bottleneck
+
+__all__ = ['resnest50', 'resnest101', 'resnest200', 'resnest269']
+
+_url_format = 'https://github.com/zhanghang1989/ResNeSt/releases/download/weights_step1/{}-{}.pth'
+
+_model_sha256 = {name: checksum for checksum, name in [
+    ('528c19ca', 'resnest50'),
+    ('22405ba7', 'resnest101'),
+    ('75117900', 'resnest200'),
+    ('0cc87c48', 'resnest269'),
+    ]}
+
+def short_hash(name):
+    if name not in _model_sha256:
+        raise ValueError('Pretrained model for {name} is not available.'.format(name=name))
+    return _model_sha256[name][:8]
+
+resnest_model_urls = {name: _url_format.format(name, short_hash(name)) for
+    name in _model_sha256.keys()
+}
+
+def resnest50(pretrained=False, root='~/.encoding/models', **kwargs):
+    model = ResNet(Bottleneck, [3, 4, 6, 3],
+                   radix=2, groups=1, bottleneck_width=64,
+                   deep_stem=True, stem_width=32, avg_down=True,
+                   avd=True, avd_first=False, **kwargs)
+    if pretrained:
+        model.load_state_dict(torch.hub.load_state_dict_from_url(
+            resnest_model_urls['resnest50'], progress=True, check_hash=True))
+    return model
+
+def resnest101(pretrained=False, root='~/.encoding/models', **kwargs):
+    model = ResNet(Bottleneck, [3, 4, 23, 3],
+                   radix=2, groups=1, bottleneck_width=64,
+                   deep_stem=True, stem_width=64, avg_down=True,
+                   avd=True, avd_first=False, **kwargs)
+    if pretrained:
+        model.load_state_dict(torch.hub.load_state_dict_from_url(
+            resnest_model_urls['resnest101'], progress=True, check_hash=True))
+    return model
+
+def resnest200(pretrained=False, root='~/.encoding/models', **kwargs):
+    model = ResNet(Bottleneck, [3, 24, 36, 3],
+                   radix=2, groups=1, bottleneck_width=64,
+                   deep_stem=True, stem_width=64, avg_down=True,
+                   avd=True, avd_first=False, **kwargs)
+    if pretrained:
+        model.load_state_dict(torch.hub.load_state_dict_from_url(
+            resnest_model_urls['resnest200'], progress=True, check_hash=True))
+    return model
+
+def resnest269(pretrained=False, root='~/.encoding/models', **kwargs):
+    model = ResNet(Bottleneck, [3, 30, 48, 8],
+                   radix=2, groups=1, bottleneck_width=64,
+                   deep_stem=True, stem_width=64, avg_down=True,
+                   avd=True, avd_first=False, **kwargs)
+    if pretrained:
+        model.load_state_dict(torch.hub.load_state_dict_from_url(
+            resnest_model_urls['resnest269'], progress=True, check_hash=True))
+    return model

core/arch_resnest/resnet.py

@@ -0,0 +1,308 @@
+##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+## Created by: Hang Zhang
+## Email: [email protected]
+## Copyright (c) 2020
+##
+## LICENSE file in the root directory of this source tree 
+##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+"""ResNet variants"""
+import math
+import torch
+import torch.nn as nn
+
+from .splat import SplAtConv2d
+
+__all__ = ['ResNet', 'Bottleneck']
+
+class DropBlock2D(object):
+    def __init__(self, *args, **kwargs):
+        raise NotImplementedError
+
+class GlobalAvgPool2d(nn.Module):
+    def __init__(self):
+        """Global average pooling over the input's spatial dimensions"""
+        super(GlobalAvgPool2d, self).__init__()
+
+    def forward(self, inputs):
+        return nn.functional.adaptive_avg_pool2d(inputs, 1).view(inputs.size(0), -1)
+
+class Bottleneck(nn.Module):
+    """ResNet Bottleneck
+    """
+    # pylint: disable=unused-argument
+    expansion = 4
+    def __init__(self, inplanes, planes, stride=1, downsample=None,
+                 radix=1, cardinality=1, bottleneck_width=64,
+                 avd=False, avd_first=False, dilation=1, is_first=False,
+                 rectified_conv=False, rectify_avg=False,
+                 norm_layer=None, dropblock_prob=0.0, last_gamma=False):
+        super(Bottleneck, self).__init__()
+        group_width = int(planes * (bottleneck_width / 64.)) * cardinality
+        self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False)
+        self.bn1 = norm_layer(group_width)
+        self.dropblock_prob = dropblock_prob
+        self.radix = radix
+        self.avd = avd and (stride > 1 or is_first)
+        self.avd_first = avd_first
+
+        if self.avd:
+            self.avd_layer = nn.AvgPool2d(3, stride, padding=1)
+            stride = 1
+
+        if dropblock_prob > 0.0:
+            self.dropblock1 = DropBlock2D(dropblock_prob, 3)
+            if radix == 1:
+                self.dropblock2 = DropBlock2D(dropblock_prob, 3)
+            self.dropblock3 = DropBlock2D(dropblock_prob, 3)
+
+        if radix >= 1:
+            self.conv2 = SplAtConv2d(
+                group_width, group_width, kernel_size=3,
+                stride=stride, padding=dilation,
+                dilation=dilation, groups=cardinality, bias=False,
+                radix=radix, rectify=rectified_conv,
+                rectify_avg=rectify_avg,
+                norm_layer=norm_layer,
+                dropblock_prob=dropblock_prob)
+        elif rectified_conv:
+            from rfconv import RFConv2d
+            self.conv2 = RFConv2d(
+                group_width, group_width, kernel_size=3, stride=stride,
+                padding=dilation, dilation=dilation,
+                groups=cardinality, bias=False,
+                average_mode=rectify_avg)
+            self.bn2 = norm_layer(group_width)
+        else:
+            self.conv2 = nn.Conv2d(
+                group_width, group_width, kernel_size=3, stride=stride,
+                padding=dilation, dilation=dilation,
+                groups=cardinality, bias=False)
+            self.bn2 = norm_layer(group_width)
+
+        self.conv3 = nn.Conv2d(
+            group_width, planes * 4, kernel_size=1, bias=False)
+        self.bn3 = norm_layer(planes*4)
+
+        if last_gamma:
+            from torch.nn.init import zeros_
+            zeros_(self.bn3.weight)
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = downsample
+        self.dilation = dilation
+        self.stride = stride
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        if self.dropblock_prob > 0.0:
+            out = self.dropblock1(out)
+        out = self.relu(out)
+
+        if self.avd and self.avd_first:
+            out = self.avd_layer(out)
+
+        out = self.conv2(out)
+        if self.radix == 0:
+            out = self.bn2(out)
+            if self.dropblock_prob > 0.0:
+                out = self.dropblock2(out)
+            out = self.relu(out)
+
+        if self.avd and not self.avd_first:
+            out = self.avd_layer(out)
+
+        out = self.conv3(out)
+        out = self.bn3(out)
+        if self.dropblock_prob > 0.0:
+            out = self.dropblock3(out)
+
+        if self.downsample is not None:
+            residual = self.downsample(x)
+
+        out += residual
+        out = self.relu(out)
+
+        return out
+
+class ResNet(nn.Module):
+    """ResNet Variants
+
+    Parameters
+    ----------
+    block : Block
+        Class for the residual block. Options are BasicBlockV1, BottleneckV1.
+    layers : list of int
+        Numbers of layers in each block
+    classes : int, default 1000
+        Number of classification classes.
+    dilated : bool, default False
+        Applying dilation strategy to pretrained ResNet yielding a stride-8 model,
+        typically used in Semantic Segmentation.
+    norm_layer : object
+        Normalization layer used in backbone network (default: :class:`mxnet.gluon.nn.BatchNorm`;
+        for Synchronized Cross-GPU BachNormalization).
+
+    Reference:
+
+        - He, Kaiming, et al. "Deep residual learning for image recognition." Proceedings of the IEEE conference on computer vision and pattern recognition. 2016.
+
+        - Yu, Fisher, and Vladlen Koltun. "Multi-scale context aggregation by dilated convolutions."
+    """
+    # pylint: disable=unused-variable
+    def __init__(self, block, layers, radix=1, groups=1, bottleneck_width=64,
+                 num_classes=1000, dilated=False, dilation=1,
+                 deep_stem=False, stem_width=64, avg_down=False,
+                 rectified_conv=False, rectify_avg=False,
+                 avd=False, avd_first=False,
+                 final_drop=0.0, dropblock_prob=0,
+                 last_gamma=False, norm_layer=nn.BatchNorm2d):
+        self.cardinality = groups
+        self.bottleneck_width = bottleneck_width
+        # ResNet-D params
+        self.inplanes = stem_width*2 if deep_stem else 64
+        self.avg_down = avg_down
+        self.last_gamma = last_gamma
+        # ResNeSt params
+        self.radix = radix
+        self.avd = avd
+        self.avd_first = avd_first
+
+        super(ResNet, self).__init__()
+        self.rectified_conv = rectified_conv
+        self.rectify_avg = rectify_avg
+        if rectified_conv:
+            from rfconv import RFConv2d
+            conv_layer = RFConv2d
+        else:
+            conv_layer = nn.Conv2d
+        conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {}
+        if deep_stem:
+            self.conv1 = nn.Sequential(
+                conv_layer(3, stem_width, kernel_size=3, stride=2, padding=1, bias=False, **conv_kwargs),
+                norm_layer(stem_width),
+                nn.ReLU(inplace=True),
+                conv_layer(stem_width, stem_width, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),
+                norm_layer(stem_width),
+                nn.ReLU(inplace=True),
+                conv_layer(stem_width, stem_width*2, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),
+            )
+        else:
+            self.conv1 = conv_layer(3, 64, kernel_size=7, stride=2, padding=3,
+                                   bias=False, **conv_kwargs)
+        self.bn1 = norm_layer(self.inplanes)
+        self.relu = nn.ReLU(inplace=True)
+        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+        self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer, is_first=False)
+        self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer)
+        if dilated or dilation == 4:
+            self.layer3 = self._make_layer(block, 256, layers[2], stride=1,
+                                           dilation=2, norm_layer=norm_layer,
+                                           dropblock_prob=dropblock_prob)
+            self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
+                                           dilation=4, norm_layer=norm_layer,
+                                           dropblock_prob=dropblock_prob)
+        elif dilation==2:
+            self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
+                                           dilation=1, norm_layer=norm_layer,
+                                           dropblock_prob=dropblock_prob)
+            self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
+                                           dilation=2, norm_layer=norm_layer,
+                                           dropblock_prob=dropblock_prob)
+        else:
+            self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
+                                           norm_layer=norm_layer,
+                                           dropblock_prob=dropblock_prob)
+            self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
+                                           norm_layer=norm_layer,
+                                           dropblock_prob=dropblock_prob)
+
+        self.avgpool = GlobalAvgPool2d()
+        self.drop = nn.Dropout(final_drop) if final_drop > 0.0 else None
+        self.fc = nn.Linear(512 * block.expansion, num_classes)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                m.weight.data.normal_(0, math.sqrt(2. / n))
+            elif isinstance(m, norm_layer):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+
+    def _make_layer(self, block, planes, blocks, stride=1, dilation=1, norm_layer=None,
+                    dropblock_prob=0.0, is_first=True):
+        downsample = None
+        if stride != 1 or self.inplanes != planes * block.expansion:
+            down_layers = []
+            if self.avg_down:
+                if dilation == 1:
+                    down_layers.append(nn.AvgPool2d(kernel_size=stride, stride=stride,
+                                                    ceil_mode=True, count_include_pad=False))
+                else:
+                    down_layers.append(nn.AvgPool2d(kernel_size=1, stride=1,
+                                                    ceil_mode=True, count_include_pad=False))
+                down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
+                                             kernel_size=1, stride=1, bias=False))
+            else:
+                down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
+                                             kernel_size=1, stride=stride, bias=False))
+            down_layers.append(norm_layer(planes * block.expansion))
+            downsample = nn.Sequential(*down_layers)
+        
+        layers = []
+        if dilation == 1 or dilation == 2:
+            layers.append(block(self.inplanes, planes, stride, downsample=downsample,
+                                radix=self.radix, cardinality=self.cardinality,
+                                bottleneck_width=self.bottleneck_width,
+                                avd=self.avd, avd_first=self.avd_first,
+                                dilation=1, is_first=is_first, rectified_conv=self.rectified_conv,
+                                rectify_avg=self.rectify_avg,
+                                norm_layer=norm_layer, dropblock_prob=dropblock_prob,
+                                last_gamma=self.last_gamma))
+        elif dilation == 4:
+            layers.append(block(self.inplanes, planes, stride, downsample=downsample,
+                                radix=self.radix, cardinality=self.cardinality,
+                                bottleneck_width=self.bottleneck_width,
+                                avd=self.avd, avd_first=self.avd_first,
+                                dilation=2, is_first=is_first, rectified_conv=self.rectified_conv,
+                                rectify_avg=self.rectify_avg,
+                                norm_layer=norm_layer, dropblock_prob=dropblock_prob,
+                                last_gamma=self.last_gamma))
+        else:
+            raise RuntimeError("=> unknown dilation size: {}".format(dilation))
+
+        self.inplanes = planes * block.expansion
+        for i in range(1, blocks):
+            layers.append(block(self.inplanes, planes,
+                                radix=self.radix, cardinality=self.cardinality,
+                                bottleneck_width=self.bottleneck_width,
+                                avd=self.avd, avd_first=self.avd_first,
+                                dilation=dilation, rectified_conv=self.rectified_conv,
+                                rectify_avg=self.rectify_avg,
+                                norm_layer=norm_layer, dropblock_prob=dropblock_prob,
+                                last_gamma=self.last_gamma))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+        x = self.maxpool(x)
+
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+
+        # print(x.size())
+
+        x = self.avgpool(x)
+        #x = x.view(x.size(0), -1)
+        x = torch.flatten(x, 1)
+        if self.drop:
+            x = self.drop(x)
+        x = self.fc(x)
+
+        return x

core/arch_resnest/splat.py

@@ -0,0 +1,99 @@
+"""Split-Attention"""
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.nn import Conv2d, Module, Linear, BatchNorm2d, ReLU
+from torch.nn.modules.utils import _pair
+
+__all__ = ['SplAtConv2d']
+
+class SplAtConv2d(Module):
+    """Split-Attention Conv2d
+    """
+    def __init__(self, in_channels, channels, kernel_size, stride=(1, 1), padding=(0, 0),
+                 dilation=(1, 1), groups=1, bias=True,
+                 radix=2, reduction_factor=4,
+                 rectify=False, rectify_avg=False, norm_layer=None,
+                 dropblock_prob=0.0, **kwargs):
+        super(SplAtConv2d, self).__init__()
+        padding = _pair(padding)
+        self.rectify = rectify and (padding[0] > 0 or padding[1] > 0)
+        self.rectify_avg = rectify_avg
+        inter_channels = max(in_channels*radix//reduction_factor, 32)
+        self.radix = radix
+        self.cardinality = groups
+        self.channels = channels
+        self.dropblock_prob = dropblock_prob
+        if self.rectify:
+            from rfconv import RFConv2d
+            self.conv = RFConv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
+                                 groups=groups*radix, bias=bias, average_mode=rectify_avg, **kwargs)
+        else:
+            self.conv = Conv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
+                               groups=groups*radix, bias=bias, **kwargs)
+        self.use_bn = norm_layer is not None
+        if self.use_bn:
+            self.bn0 = norm_layer(channels*radix)
+        self.relu = ReLU(inplace=True)
+        self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality)
+        if self.use_bn:
+            self.bn1 = norm_layer(inter_channels)
+        self.fc2 = Conv2d(inter_channels, channels*radix, 1, groups=self.cardinality)
+        if dropblock_prob > 0.0:
+            self.dropblock = DropBlock2D(dropblock_prob, 3)
+        self.rsoftmax = rSoftMax(radix, groups)
+
+    def forward(self, x):
+        x = self.conv(x)
+        if self.use_bn:
+            x = self.bn0(x)
+        if self.dropblock_prob > 0.0:
+            x = self.dropblock(x)
+        x = self.relu(x)
+
+        batch, rchannel = x.shape[:2]
+        if self.radix > 1:
+            if torch.__version__ < '1.5':
+                splited = torch.split(x, int(rchannel//self.radix), dim=1)
+            else:
+                splited = torch.split(x, rchannel//self.radix, dim=1)
+            gap = sum(splited) 
+        else:
+            gap = x
+        gap = F.adaptive_avg_pool2d(gap, 1)
+        gap = self.fc1(gap)
+
+        if self.use_bn:
+            gap = self.bn1(gap)
+        gap = self.relu(gap)
+
+        atten = self.fc2(gap)
+        atten = self.rsoftmax(atten).view(batch, -1, 1, 1)
+
+        if self.radix > 1:
+            if torch.__version__ < '1.5':
+                attens = torch.split(atten, int(rchannel//self.radix), dim=1)
+            else:
+                attens = torch.split(atten, rchannel//self.radix, dim=1)
+            out = sum([att*split for (att, split) in zip(attens, splited)])
+        else:
+            out = atten * x
+        return out.contiguous()
+
+class rSoftMax(nn.Module):
+    def __init__(self, radix, cardinality):
+        super().__init__()
+        self.radix = radix
+        self.cardinality = cardinality
+
+    def forward(self, x):
+        batch = x.size(0)
+        if self.radix > 1:
+            x = x.view(batch, self.cardinality, self.radix, -1).transpose(1, 2)
+            x = F.softmax(x, dim=1)
+            x = x.reshape(batch, -1)
+        else:
+            x = torch.sigmoid(x)
+        return x
+

core/arch_resnet/resnet.py

@@ -0,0 +1,157 @@
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.model_zoo as model_zoo
+
+urls_dic = {
+    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
+    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
+    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
+    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
+    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
+}
+
+layers_dic = {
+    'resnet18' : [2, 2, 2, 2],
+    'resnet34' : [3, 4, 6, 3],
+    'resnet50' : [3, 4, 6, 3],
+    'resnet101' : [3, 4, 23, 3],
+    'resnet152' : [3, 8, 36, 3]
+}
+
+def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d:
+    """3x3 convolution with padding"""
+    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
+                     padding=dilation, groups=groups, bias=False, dilation=dilation)
+
+def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d:
+    """1x1 convolution"""
+    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
+
+class BasicBlock(nn.Module):
+    expansion: int = 1
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None, dilation=1, batch_norm_fn=nn.BatchNorm2d):
+        super(BasicBlock, self).__init__()
+        
+        self.conv1 = conv3x3(inplanes, planes, stride)
+        self.bn1 = batch_norm_fn(planes)
+        self.relu = nn.ReLU(inplace=True)
+        self.conv2 = conv3x3(planes, planes)
+        self.bn2 = batch_norm_fn(planes)
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+        identity = x
+        
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu(out)
+
+        return out
+
+class Bottleneck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None, dilation=1, batch_norm_fn=nn.BatchNorm2d):
+        super(Bottleneck, self).__init__()
+        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
+        self.bn1 = batch_norm_fn(planes)
+
+        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
+                               padding=dilation, bias=False, dilation=dilation)
+        self.bn2 = batch_norm_fn(planes)
+
+        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
+        self.bn3 = batch_norm_fn(planes * 4)
+
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = downsample
+        self.stride = stride
+        self.dilation = dilation
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+        out = self.relu(out)
+
+        out = self.conv3(out)
+        out = self.bn3(out)
+
+        if self.downsample is not None:
+            residual = self.downsample(x)
+
+        out += residual
+        out = self.relu(out)
+
+        return out
+
+class ResNet(nn.Module):
+    
+    def __init__(self, block, layers, strides=(2, 2, 2, 2), dilations=(1, 1, 1, 1), batch_norm_fn=nn.BatchNorm2d):
+        self.batch_norm_fn = batch_norm_fn
+
+        self.inplanes = 64
+        super(ResNet, self).__init__()
+        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=strides[0], padding=3,
+                               bias=False)
+        self.bn1 = self.batch_norm_fn(64)
+        self.relu = nn.ReLU(inplace=True)
+        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+        self.layer1 = self._make_layer(block, 64, layers[0], stride=1, dilation=dilations[0])
+        self.layer2 = self._make_layer(block, 128, layers[1], stride=strides[1], dilation=dilations[1])
+        self.layer3 = self._make_layer(block, 256, layers[2], stride=strides[2], dilation=dilations[2])
+        self.layer4 = self._make_layer(block, 512, layers[3], stride=strides[3], dilation=dilations[3])
+        self.inplanes = 1024
+
+        #self.avgpool = nn.AvgPool2d(7, stride=1)
+        #self.fc = nn.Linear(512 * block.expansion, 1000)
+
+    def _make_layer(self, block, planes, blocks, stride=1, dilation=1):
+        downsample = None
+        if stride != 1 or self.inplanes != planes * block.expansion:
+            downsample = nn.Sequential(
+                nn.Conv2d(self.inplanes, planes * block.expansion,
+                          kernel_size=1, stride=stride, bias=False),
+                self.batch_norm_fn(planes * block.expansion),
+            )
+
+        layers = [block(self.inplanes, planes, stride, downsample, dilation=1, batch_norm_fn=self.batch_norm_fn)]
+        self.inplanes = planes * block.expansion
+        for i in range(1, blocks):
+            layers.append(block(self.inplanes, planes, dilation=dilation, batch_norm_fn=self.batch_norm_fn))
+        
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+        x = self.maxpool(x)
+
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+
+        x = self.avgpool(x)
+        x = x.view(x.size(0), -1)
+        x = self.fc(x)
+        
+        return x
+

core/datasets.py

@@ -0,0 +1,239 @@
+import os
+import cv2
+import glob
+import torch
+
+import math
+import imageio
+import numpy as np
+
+from PIL import Image
+
+from core.aff_utils import *
+
+from tools.ai.augment_utils import *
+from tools.ai.torch_utils import one_hot_embedding
+
+from tools.general.xml_utils import read_xml
+from tools.general.json_utils import read_json
+from tools.dataset.voc_utils import get_color_map_dic
+
+class Iterator:
+    def __init__(self, loader):
+        self.loader = loader
+        self.init()
+
+    def init(self):
+        self.iterator = iter(self.loader)
+    
+    def get(self):
+        try:
+            data = next(self.iterator)
+        except StopIteration:
+            self.init()
+            data = next(self.iterator)
+        
+        return data
+
+class VOC_Dataset(torch.utils.data.Dataset):
+    def __init__(self, root_dir, domain, with_id=False, with_tags=False, with_mask=False):
+        self.root_dir = root_dir
+
+        self.image_dir = self.root_dir + 'JPEGImages/'
+        self.xml_dir = self.root_dir + 'Annotations/'
+        self.mask_dir = self.root_dir + 'SegmentationClass/'
+        
+        self.image_id_list = [image_id.strip() for image_id in open('./data/%s.txt'%domain).readlines()]
+        
+        self.with_id = with_id
+        self.with_tags = with_tags
+        self.with_mask = with_mask
+
+    def __len__(self):
+        return len(self.image_id_list)
+
+    def get_image(self, image_id):
+        image = Image.open(self.image_dir + image_id + '.jpg').convert('RGB')
+        return image
+
+    def get_mask(self, image_id):
+        mask_path = self.mask_dir + image_id + '.png'
+        if os.path.isfile(mask_path):
+            mask = Image.open(mask_path)
+        else:
+            mask = None
+        return mask
+
+    def get_tags(self, image_id):
+        _, tags = read_xml(self.xml_dir + image_id + '.xml')
+        return tags
+    
+    def __getitem__(self, index):
+        image_id = self.image_id_list[index]
+
+        data_list = [self.get_image(image_id)]
+
+        if self.with_id:
+            data_list.append(image_id)
+
+        if self.with_tags:
+            data_list.append(self.get_tags(image_id))
+
+        if self.with_mask:
+            data_list.append(self.get_mask(image_id))
+        
+        return data_list
+
+class VOC_Dataset_For_Classification(VOC_Dataset):
+    def __init__(self, root_dir, domain, transform=None):
+        super().__init__(root_dir, domain, with_tags=True)
+        self.transform = transform
+
+        data = read_json('./data/VOC_2012.json')
+
+        self.class_dic = data['class_dic']
+        self.classes = data['classes']
+
+    def __getitem__(self, index):
+        image, tags = super().__getitem__(index)
+
+        if self.transform is not None:
+            image = self.transform(image)
+
+        label = one_hot_embedding([self.class_dic[tag] for tag in tags], self.classes)
+        return image, label
+
+class VOC_Dataset_For_Segmentation(VOC_Dataset):
+    def __init__(self, root_dir, domain, transform=None):
+        super().__init__(root_dir, domain, with_mask=True)
+        self.transform = transform
+
+        cmap_dic, _, class_names = get_color_map_dic()
+        self.colors = np.asarray([cmap_dic[class_name] for class_name in class_names])
+    
+    def __getitem__(self, index):
+        image, mask = super().__getitem__(index)
+
+        if self.transform is not None:
+            input_dic = {'image':image, 'mask':mask}
+            output_dic = self.transform(input_dic)
+
+            image = output_dic['image']
+            mask = output_dic['mask']
+        
+        return image, mask
+
+class VOC_Dataset_For_Evaluation(VOC_Dataset):
+    def __init__(self, root_dir, domain, transform=None):
+        super().__init__(root_dir, domain, with_id=True, with_mask=True)
+        self.transform = transform
+
+        cmap_dic, _, class_names = get_color_map_dic()
+        self.colors = np.asarray([cmap_dic[class_name] for class_name in class_names])
+
+    def __getitem__(self, index):
+        image, image_id, mask = super().__getitem__(index)
+
+        if self.transform is not None:
+            input_dic = {'image':image, 'mask':mask}
+            output_dic = self.transform(input_dic)
+
+            image = output_dic['image']
+            mask = output_dic['mask']
+        
+        return image, image_id, mask
+
+class VOC_Dataset_For_WSSS(VOC_Dataset):
+    def __init__(self, root_dir, domain, pred_dir, transform=None):
+        super().__init__(root_dir, domain, with_id=True)
+        self.pred_dir = pred_dir
+        self.transform = transform
+
+        cmap_dic, _, class_names = get_color_map_dic()
+        self.colors = np.asarray([cmap_dic[class_name] for class_name in class_names])
+    
+    def __getitem__(self, index):
+        image, image_id = super().__getitem__(index)
+        mask = Image.open(self.pred_dir + image_id + '.png')
+        
+        if self.transform is not None:
+            input_dic = {'image':image, 'mask':mask}
+            output_dic = self.transform(input_dic)
+
+            image = output_dic['image']
+            mask = output_dic['mask']
+        
+        return image, mask
+
+class VOC_Dataset_For_Testing_CAM(VOC_Dataset):
+    def __init__(self, root_dir, domain, transform=None):
+        super().__init__(root_dir, domain, with_tags=True, with_mask=True)
+        self.transform = transform
+
+        cmap_dic, _, class_names = get_color_map_dic()
+        self.colors = np.asarray([cmap_dic[class_name] for class_name in class_names])
+        
+        data = read_json('./data/VOC_2012.json')
+
+        self.class_dic = data['class_dic']
+        self.classes = data['classes']
+
+    def __getitem__(self, index):
+        image, tags, mask = super().__getitem__(index)
+
+        if self.transform is not None:
+            input_dic = {'image':image, 'mask':mask}
+            output_dic = self.transform(input_dic)
+
+            image = output_dic['image']
+            mask = output_dic['mask']
+        
+        label = one_hot_embedding([self.class_dic[tag] for tag in tags], self.classes)
+        return image, label, mask
+
+class VOC_Dataset_For_Making_CAM(VOC_Dataset):
+    def __init__(self, root_dir, domain):
+        super().__init__(root_dir, domain, with_id=True, with_tags=True, with_mask=True)
+
+        cmap_dic, _, class_names = get_color_map_dic()
+        self.colors = np.asarray([cmap_dic[class_name] for class_name in class_names])
+        
+        data = read_json('./data/VOC_2012.json')
+
+        self.class_names = np.asarray(class_names[1:21])
+        self.class_dic = data['class_dic']
+        self.classes = data['classes']
+
+    def __getitem__(self, index):
+        image, image_id, tags, mask = super().__getitem__(index)
+
+        label = one_hot_embedding([self.class_dic[tag] for tag in tags], self.classes)
+        return image, image_id, label, mask
+
+class VOC_Dataset_For_Affinity(VOC_Dataset):
+    def __init__(self, root_dir, domain, path_index, label_dir, transform=None):
+        super().__init__(root_dir, domain, with_id=True)
+
+        data = read_json('./data/VOC_2012.json')
+
+        self.class_dic = data['class_dic']
+        self.classes = data['classes']
+
+        self.transform = transform
+
+        self.label_dir = label_dir
+        self.path_index = path_index
+
+        self.extract_aff_lab_func = GetAffinityLabelFromIndices(self.path_index.src_indices, self.path_index.dst_indices)
+
+    def __getitem__(self, idx):
+        image, image_id = super().__getitem__(idx)
+
+        label = imageio.imread(self.label_dir + image_id + '.png')
+        label = Image.fromarray(label)
+        
+        output_dic = self.transform({'image':image, 'mask':label})
+        image, label = output_dic['image'], output_dic['mask']
+        
+        return image, self.extract_aff_lab_func(label)
+

core/deeplab_utils.py

@@ -0,0 +1,126 @@
+# Copyright (C) 2021 * Ltd. All rights reserved.
+# author : Sanghyeon Jo <[email protected]>
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class ASPPModule(nn.Module):
+    def __init__(self, inplanes, planes, kernel_size, padding, dilation, norm_fn=None):
+        super().__init__()
+        self.atrous_conv = nn.Conv2d(inplanes, planes, kernel_size=kernel_size, stride=1, padding=padding, dilation=dilation, bias=False)
+        self.bn = norm_fn(planes)
+        self.relu = nn.ReLU(inplace=True)
+
+        self.initialize([self.atrous_conv, self.bn])
+
+    def forward(self, x):
+        x = self.atrous_conv(x)
+        x = self.bn(x)
+        return self.relu(x)
+
+    def initialize(self, modules):
+        for m in modules:
+            if isinstance(m, nn.Conv2d):
+                torch.nn.init.kaiming_normal_(m.weight)
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+
+class ASPP(nn.Module):
+    def __init__(self, output_stride, norm_fn):
+        super().__init__()
+
+        inplanes = 2048
+
+        if output_stride == 16:
+            dilations = [1, 6, 12, 18]
+        elif output_stride == 8:
+            dilations = [1, 12, 24, 36]
+        
+        self.aspp1 = ASPPModule(inplanes, 256, 1, padding=0, dilation=dilations[0], norm_fn=norm_fn)
+        self.aspp2 = ASPPModule(inplanes, 256, 3, padding=dilations[1], dilation=dilations[1], norm_fn=norm_fn)
+        self.aspp3 = ASPPModule(inplanes, 256, 3, padding=dilations[2], dilation=dilations[2], norm_fn=norm_fn)
+        self.aspp4 = ASPPModule(inplanes, 256, 3, padding=dilations[3], dilation=dilations[3], norm_fn=norm_fn)
+
+        self.global_avg_pool = nn.Sequential(
+            nn.AdaptiveAvgPool2d((1, 1)),
+            nn.Conv2d(inplanes, 256, 1, stride=1, bias=False),
+            norm_fn(256),
+            nn.ReLU(inplace=True),
+        )
+
+        self.conv1 = nn.Conv2d(1280, 256, 1, bias=False)
+        self.bn1 = norm_fn(256)
+        self.relu = nn.ReLU(inplace=True)
+        self.dropout = nn.Dropout(0.5)
+        
+        self.initialize([self.conv1, self.bn1] + list(self.global_avg_pool.modules()))
+    
+    def forward(self, x):
+        x1 = self.aspp1(x)
+        x2 = self.aspp2(x)
+        x3 = self.aspp3(x)
+        x4 = self.aspp4(x)
+
+        x5 = self.global_avg_pool(x)
+        x5 = F.interpolate(x5, size=x4.size()[2:], mode='bilinear', align_corners=True)
+
+        x = torch.cat((x1, x2, x3, x4, x5), dim=1)
+
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+        x = self.dropout(x)
+
+        return x
+
+    def initialize(self, modules):
+        for m in modules:
+            if isinstance(m, nn.Conv2d):
+                torch.nn.init.kaiming_normal_(m.weight)
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+
+class Decoder(nn.Module):
+    def __init__(self, num_classes, low_level_inplanes, norm_fn):
+        super().__init__()
+        
+        self.conv1 = nn.Conv2d(low_level_inplanes, 48, 1, bias=False)
+        self.bn1 = norm_fn(48)
+        self.relu = nn.ReLU(inplace=True)
+        
+        self.classifier = nn.Sequential(
+            nn.Conv2d(304, 256, kernel_size=3, stride=1, padding=1, bias=False),
+            norm_fn(256),
+            nn.ReLU(inplace=True),
+            nn.Dropout(0.5),
+
+            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=False),
+            norm_fn(256),
+            nn.ReLU(inplace=True),
+            nn.Dropout(0.1),
+            nn.Conv2d(256, num_classes, kernel_size=1, stride=1)
+        )
+
+        self.initialize([self.conv1, self.bn1] + list(self.classifier.modules()))
+
+    def forward(self, x, x_low_level):
+        x_low_level = self.conv1(x_low_level)
+        x_low_level = self.bn1(x_low_level)
+        x_low_level = self.relu(x_low_level)
+
+        x = F.interpolate(x, size=x_low_level.size()[2:], mode='bilinear', align_corners=True)
+        x = torch.cat((x, x_low_level), dim=1)
+        x = self.classifier(x)
+
+        return x
+
+    def initialize(self, modules):
+        for m in modules:
+            if isinstance(m, nn.Conv2d):
+                torch.nn.init.kaiming_normal_(m.weight)
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()

core/networks.py

@@ -0,0 +1,355 @@
+# Copyright (C) 2021 * Ltd. All rights reserved.
+# author : Sanghyeon Jo <[email protected]>
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torchvision import models
+import torch.utils.model_zoo as model_zoo
+
+from .arch_resnet import resnet
+from .arch_resnest import resnest
+from .abc_modules import ABC_Model
+
+from .deeplab_utils import ASPP, Decoder
+from .aff_utils import PathIndex
+from .puzzle_utils import tile_features, merge_features
+
+from tools.ai.torch_utils import resize_for_tensors
+
+#######################################################################
+# Normalization
+#######################################################################
+from .sync_batchnorm.batchnorm import SynchronizedBatchNorm2d
+
+class FixedBatchNorm(nn.BatchNorm2d):
+    def forward(self, x):
+        return F.batch_norm(x, self.running_mean, self.running_var, self.weight, self.bias, training=False, eps=self.eps)
+
+def group_norm(features):
+    return nn.GroupNorm(4, features)
+#######################################################################
+
+class Backbone(nn.Module, ABC_Model):
+    def __init__(self, model_name, state_path, num_classes=20, mode='fix', segmentation=False):
+        super().__init__()
+
+        self.mode = mode
+
+        if self.mode == 'fix': 
+            self.norm_fn = FixedBatchNorm
+        else:
+            self.norm_fn = nn.BatchNorm2d
+        
+        if 'resnet' in model_name:
+            self.model = resnet.ResNet(resnet.Bottleneck, resnet.layers_dic[model_name], strides=(2, 2, 2, 1), batch_norm_fn=self.norm_fn)
+
+            state_dict = torch.load(state_path)
+            self.model.load_state_dict(state_dict, strict=False)
+        else:
+            if segmentation:
+                dilation, dilated = 4, True
+            else:
+                dilation, dilated = 2, False
+
+            self.model = eval("resnest." + model_name)(pretrained=True, dilated=dilated, dilation=dilation, norm_layer=self.norm_fn)
+
+            del self.model.avgpool
+            del self.model.fc
+
+        self.stage1 = nn.Sequential(self.model.conv1, 
+                                    self.model.bn1, 
+                                    self.model.relu, 
+                                    self.model.maxpool)
+        self.stage2 = nn.Sequential(self.model.layer1)
+        self.stage3 = nn.Sequential(self.model.layer2)
+        self.stage4 = nn.Sequential(self.model.layer3)
+        self.stage5 = nn.Sequential(self.model.layer4)
+
+class Classifier(Backbone):
+    def __init__(self, model_name, state_path, num_classes=20, mode='fix'):
+        super().__init__(model_name, state_path, num_classes, mode)
+        
+        self.classifier = nn.Conv2d(2048, num_classes, 1, bias=False)
+        self.num_classes = num_classes
+
+        self.initialize([self.classifier])
+    
+    def forward(self, x, with_cam=False):
+        x = self.stage1(x)
+        x = self.stage2(x)
+        x = self.stage3(x)
+        x = self.stage4(x)
+        x = self.stage5(x)
+        
+        if with_cam:
+            features = self.classifier(x)
+            logits = self.global_average_pooling_2d(features)
+            return logits, features
+        else:
+            x = self.global_average_pooling_2d(x, keepdims=True) 
+            logits = self.classifier(x).view(-1, self.num_classes)
+            return logits
+
+class Classifier_For_Positive_Pooling(Backbone):
+    def __init__(self, model_name, num_classes=20, mode='fix'):
+        super().__init__(model_name, num_classes, mode)
+        
+        self.classifier = nn.Conv2d(2048, num_classes, 1, bias=False)
+        self.num_classes = num_classes
+        
+        self.initialize([self.classifier])
+    
+    def forward(self, x, with_cam=False):
+        x = self.stage1(x)
+        x = self.stage2(x)
+        x = self.stage3(x)
+        x = self.stage4(x)
+        x = self.stage5(x)
+        
+        if with_cam:
+            features = self.classifier(x)
+            logits = self.global_average_pooling_2d(features)
+            return logits, features
+        else:
+            x = self.global_average_pooling_2d(x, keepdims=True) 
+            logits = self.classifier(x).view(-1, self.num_classes)
+            return logits
+
+class Classifier_For_Puzzle(Classifier):
+    def __init__(self, model_name, num_classes=20, mode='fix'):
+        super().__init__(model_name, num_classes, mode)
+        
+    def forward(self, x, num_pieces=1, level=-1):
+        batch_size = x.size()[0]
+        
+        output_dic = {}
+        layers = [self.stage1, self.stage2, self.stage3, self.stage4, self.stage5, self.classifier]
+
+        for l, layer in enumerate(layers):
+            l += 1
+            if level == l:
+                x = tile_features(x, num_pieces)
+
+            x = layer(x)
+            output_dic['stage%d'%l] = x
+        
+        output_dic['logits'] = self.global_average_pooling_2d(output_dic['stage6'])
+
+        for l in range(len(layers)):
+            l += 1
+            if l >= level:
+                output_dic['stage%d'%l] = merge_features(output_dic['stage%d'%l], num_pieces, batch_size)
+
+        if level is not None:
+            output_dic['merged_logits'] = self.global_average_pooling_2d(output_dic['stage6'])
+
+        return output_dic
+        
+class AffinityNet(Backbone):
+    def __init__(self, model_name, path_index=None):
+        super().__init__(model_name, None, 'fix')
+
+        if '50' in model_name:
+            fc_edge1_features = 64
+        else:
+            fc_edge1_features = 128
+
+        self.fc_edge1 = nn.Sequential(
+            nn.Conv2d(fc_edge1_features, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge2 = nn.Sequential(
+            nn.Conv2d(256, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge3 = nn.Sequential(
+            nn.Conv2d(512, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge4 = nn.Sequential(
+            nn.Conv2d(1024, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.Upsample(scale_factor=4, mode='bilinear', align_corners=False),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge5 = nn.Sequential(
+            nn.Conv2d(2048, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.Upsample(scale_factor=4, mode='bilinear', align_corners=False),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge6 = nn.Conv2d(160, 1, 1, bias=True)
+
+        self.backbone = nn.ModuleList([self.stage1, self.stage2, self.stage3, self.stage4, self.stage5])
+        self.edge_layers = nn.ModuleList([self.fc_edge1, self.fc_edge2, self.fc_edge3, self.fc_edge4, self.fc_edge5, self.fc_edge6])
+
+        if path_index is not None:
+            self.path_index = path_index
+            self.n_path_lengths = len(self.path_index.path_indices)
+            for i, pi in enumerate(self.path_index.path_indices):
+                self.register_buffer("path_indices_" + str(i), torch.from_numpy(pi))
+    
+    def train(self, mode=True):
+        super().train(mode)
+        self.backbone.eval()
+
+    def forward(self, x, with_affinity=False):
+        x1 = self.stage1(x).detach()
+        x2 = self.stage2(x1).detach()
+        x3 = self.stage3(x2).detach()
+        x4 = self.stage4(x3).detach()
+        x5 = self.stage5(x4).detach()
+        
+        edge1 = self.fc_edge1(x1)
+        edge2 = self.fc_edge2(x2)
+        edge3 = self.fc_edge3(x3)[..., :edge2.size(2), :edge2.size(3)]
+        edge4 = self.fc_edge4(x4)[..., :edge2.size(2), :edge2.size(3)]
+        edge5 = self.fc_edge5(x5)[..., :edge2.size(2), :edge2.size(3)]
+
+        edge = self.fc_edge6(torch.cat([edge1, edge2, edge3, edge4, edge5], dim=1))
+
+        if with_affinity:
+            return edge, self.to_affinity(torch.sigmoid(edge))
+        else:
+            return edge
+
+    def get_edge(self, x, image_size=512, stride=4):
+        feat_size = (x.size(2)-1)//stride+1, (x.size(3)-1)//stride+1
+
+        x = F.pad(x, [0, image_size-x.size(3), 0, image_size-x.size(2)])
+        edge_out = self.forward(x)
+        edge_out = edge_out[..., :feat_size[0], :feat_size[1]]
+        edge_out = torch.sigmoid(edge_out[0]/2 + edge_out[1].flip(-1)/2)
+        
+        return edge_out
+    
+    """
+    aff = self.to_affinity(torch.sigmoid(edge_out))
+    pos_aff_loss = (-1) * torch.log(aff + 1e-5)
+    neg_aff_loss = (-1) * torch.log(1. + 1e-5 - aff)
+    """
+    def to_affinity(self, edge):
+        aff_list = []
+        edge = edge.view(edge.size(0), -1)
+        
+        for i in range(self.n_path_lengths):
+            ind = self._buffers["path_indices_" + str(i)]
+            ind_flat = ind.view(-1)
+            dist = torch.index_select(edge, dim=-1, index=ind_flat)
+            dist = dist.view(dist.size(0), ind.size(0), ind.size(1), ind.size(2))
+            aff = torch.squeeze(1 - F.max_pool2d(dist, (dist.size(2), 1)), dim=2)
+            aff_list.append(aff)
+        aff_cat = torch.cat(aff_list, dim=1)
+        return aff_cat
+
+class DeepLabv3_Plus(Backbone):
+    def __init__(self, model_name, num_classes=21, mode='fix', use_group_norm=False):
+        super().__init__(model_name, num_classes, mode, segmentation=False)
+        
+        if use_group_norm:
+            norm_fn_for_extra_modules = group_norm
+        else:
+            norm_fn_for_extra_modules = self.norm_fn
+        
+        self.aspp = ASPP(output_stride=16, norm_fn=norm_fn_for_extra_modules)
+        self.decoder = Decoder(num_classes, 256, norm_fn_for_extra_modules)
+        
+    def forward(self, x, with_cam=False):
+        inputs = x
+
+        x = self.stage1(x)
+        x = self.stage2(x)
+        x_low_level = x
+        
+        x = self.stage3(x)
+        x = self.stage4(x)
+        x = self.stage5(x)
+        
+        x = self.aspp(x)
+        x = self.decoder(x, x_low_level)
+        x = resize_for_tensors(x, inputs.size()[2:], align_corners=True)
+
+        return x
+
+class Seg_Model(Backbone):
+    def __init__(self, model_name, num_classes=21):
+        super().__init__(model_name, num_classes, mode='fix', segmentation=False)
+        
+        self.classifier = nn.Conv2d(2048, num_classes, 1, bias=False)
+    
+    def forward(self, inputs):
+        x = self.stage1(inputs)
+        x = self.stage2(x)
+        x = self.stage3(x)
+        x = self.stage4(x)
+        x = self.stage5(x)
+        
+        logits = self.classifier(x)
+        # logits = resize_for_tensors(logits, inputs.size()[2:], align_corners=False)
+        
+        return logits
+
+class CSeg_Model(Backbone):
+    def __init__(self, model_name, num_classes=21):
+        super().__init__(model_name, num_classes, 'fix')
+
+        if '50' in model_name:
+            fc_edge1_features = 64
+        else:
+            fc_edge1_features = 128
+
+        self.fc_edge1 = nn.Sequential(
+            nn.Conv2d(fc_edge1_features, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge2 = nn.Sequential(
+            nn.Conv2d(256, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge3 = nn.Sequential(
+            nn.Conv2d(512, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge4 = nn.Sequential(
+            nn.Conv2d(1024, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.Upsample(scale_factor=4, mode='bilinear', align_corners=False),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge5 = nn.Sequential(
+            nn.Conv2d(2048, 32, 1, bias=False),
+            nn.GroupNorm(4, 32),
+            nn.Upsample(scale_factor=4, mode='bilinear', align_corners=False),
+            nn.ReLU(inplace=True),
+        )
+        self.fc_edge6 = nn.Conv2d(160, num_classes, 1, bias=True)
+
+    def forward(self, x):
+        x1 = self.stage1(x)
+        x2 = self.stage2(x1)
+        x3 = self.stage3(x2)
+        x4 = self.stage4(x3)
+        x5 = self.stage5(x4)
+        
+        edge1 = self.fc_edge1(x1)
+        edge2 = self.fc_edge2(x2)
+        edge3 = self.fc_edge3(x3)[..., :edge2.size(2), :edge2.size(3)]
+        edge4 = self.fc_edge4(x4)[..., :edge2.size(2), :edge2.size(3)]
+        edge5 = self.fc_edge5(x5)[..., :edge2.size(2), :edge2.size(3)]
+
+        logits = self.fc_edge6(torch.cat([edge1, edge2, edge3, edge4, edge5], dim=1))
+        # logits = resize_for_tensors(logits, x.size()[2:], align_corners=True)
+        
+        return logits

core/puzzle_utils.py

@@ -0,0 +1,69 @@
+import math
+
+import torch
+import torch.nn.functional as F
+
+def tile_features(features, num_pieces):
+    _, _, h, w = features.size()
+
+    num_pieces_per_line = int(math.sqrt(num_pieces))
+    
+    h_per_patch = h // num_pieces_per_line
+    w_per_patch = w // num_pieces_per_line
+    
+    """
+    +-----+-----+
+    |  1  |  2  |
+    +-----+-----+
+    |  3  |  4  |
+    +-----+-----+
+
+    +-----+-----+-----+-----+
+    |  1  |  2  |  3  |  4  |
+    +-----+-----+-----+-----+
+    """
+    patches = []
+    for splitted_features in torch.split(features, h_per_patch, dim=2):
+        for patch in torch.split(splitted_features, w_per_patch, dim=3):
+            patches.append(patch)
+    
+    return torch.cat(patches, dim=0)
+
+def merge_features(features, num_pieces, batch_size):
+    """
+    +-----+-----+-----+-----+
+    |  1  |  2  |  3  |  4  |
+    +-----+-----+-----+-----+
+    
+    +-----+-----+
+    |  1  |  2  |
+    +-----+-----+
+    |  3  |  4  |
+    +-----+-----+
+    """
+    features_list = list(torch.split(features, batch_size))
+    num_pieces_per_line = int(math.sqrt(num_pieces))
+    
+    index = 0
+    ext_h_list = []
+
+    for _ in range(num_pieces_per_line):
+
+        ext_w_list = []
+        for _ in range(num_pieces_per_line):
+            ext_w_list.append(features_list[index])
+            index += 1
+        
+        ext_h_list.append(torch.cat(ext_w_list, dim=3))
+
+    features = torch.cat(ext_h_list, dim=2)
+    return features
+
+def puzzle_module(x, func_list, num_pieces):
+    tiled_x = tile_features(x, num_pieces)
+
+    for func in func_list:
+        tiled_x = func(tiled_x)
+        
+    merged_x = merge_features(tiled_x, num_pieces, x.size()[0])
+    return merged_x

core/sync_batchnorm/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# File   : __init__.py
+# Author : Jiayuan Mao
+# Email  : [email protected]
+# Date   : 27/01/2018
+#
+# This file is part of Synchronized-BatchNorm-PyTorch.
+# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
+# Distributed under MIT License.
+
+from .batchnorm import SynchronizedBatchNorm1d, SynchronizedBatchNorm2d, SynchronizedBatchNorm3d
+from .replicate import DataParallelWithCallback, patch_replication_callback

core/sync_batchnorm/batchnorm.py

@@ -0,0 +1,282 @@
+# -*- coding: utf-8 -*-
+# File   : batchnorm.py
+# Author : Jiayuan Mao
+# Email  : [email protected]
+# Date   : 27/01/2018
+#
+# This file is part of Synchronized-BatchNorm-PyTorch.
+# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
+# Distributed under MIT License.
+
+import collections
+
+import torch
+import torch.nn.functional as F
+
+from torch.nn.modules.batchnorm import _BatchNorm
+from torch.nn.parallel._functions import ReduceAddCoalesced, Broadcast
+
+from .comm import SyncMaster
+
+__all__ = ['SynchronizedBatchNorm1d', 'SynchronizedBatchNorm2d', 'SynchronizedBatchNorm3d']
+
+
+def _sum_ft(tensor):
+    """sum over the first and last dimention"""
+    return tensor.sum(dim=0).sum(dim=-1)
+
+
+def _unsqueeze_ft(tensor):
+    """add new dementions at the front and the tail"""
+    return tensor.unsqueeze(0).unsqueeze(-1)
+
+
+_ChildMessage = collections.namedtuple('_ChildMessage', ['sum', 'ssum', 'sum_size'])
+_MasterMessage = collections.namedtuple('_MasterMessage', ['sum', 'inv_std'])
+
+
+class _SynchronizedBatchNorm(_BatchNorm):
+    def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True):
+        super(_SynchronizedBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine)
+
+        self._sync_master = SyncMaster(self._data_parallel_master)
+
+        self._is_parallel = False
+        self._parallel_id = None
+        self._slave_pipe = None
+
+    def forward(self, input):
+        # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation.
+        if not (self._is_parallel and self.training):
+            return F.batch_norm(
+                input, self.running_mean, self.running_var, self.weight, self.bias,
+                self.training, self.momentum, self.eps)
+
+        # Resize the input to (B, C, -1).
+        input_shape = input.size()
+        input = input.view(input.size(0), self.num_features, -1)
+
+        # Compute the sum and square-sum.
+        sum_size = input.size(0) * input.size(2)
+        input_sum = _sum_ft(input)
+        input_ssum = _sum_ft(input ** 2)
+
+        # Reduce-and-broadcast the statistics.
+        if self._parallel_id == 0:
+            mean, inv_std = self._sync_master.run_master(_ChildMessage(input_sum, input_ssum, sum_size))
+        else:
+            mean, inv_std = self._slave_pipe.run_slave(_ChildMessage(input_sum, input_ssum, sum_size))
+
+        # Compute the output.
+        if self.affine:
+            # MJY:: Fuse the multiplication for speed.
+            output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std * self.weight) + _unsqueeze_ft(self.bias)
+        else:
+            output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std)
+
+        # Reshape it.
+        return output.view(input_shape)
+
+    def __data_parallel_replicate__(self, ctx, copy_id):
+        self._is_parallel = True
+        self._parallel_id = copy_id
+
+        # parallel_id == 0 means master device.
+        if self._parallel_id == 0:
+            ctx.sync_master = self._sync_master
+        else:
+            self._slave_pipe = ctx.sync_master.register_slave(copy_id)
+
+    def _data_parallel_master(self, intermediates):
+        """Reduce the sum and square-sum, compute the statistics, and broadcast it."""
+
+        # Always using same "device order" makes the ReduceAdd operation faster.
+        # Thanks to:: Tete Xiao (http://tetexiao.com/)
+        intermediates = sorted(intermediates, key=lambda i: i[1].sum.get_device())
+
+        to_reduce = [i[1][:2] for i in intermediates]
+        to_reduce = [j for i in to_reduce for j in i]  # flatten
+        target_gpus = [i[1].sum.get_device() for i in intermediates]
+
+        sum_size = sum([i[1].sum_size for i in intermediates])
+        sum_, ssum = ReduceAddCoalesced.apply(target_gpus[0], 2, *to_reduce)
+        mean, inv_std = self._compute_mean_std(sum_, ssum, sum_size)
+
+        broadcasted = Broadcast.apply(target_gpus, mean, inv_std)
+
+        outputs = []
+        for i, rec in enumerate(intermediates):
+            outputs.append((rec[0], _MasterMessage(*broadcasted[i * 2:i * 2 + 2])))
+
+        return outputs
+
+    def _compute_mean_std(self, sum_, ssum, size):
+        """Compute the mean and standard-deviation with sum and square-sum. This method
+        also maintains the moving average on the master device."""
+        assert size > 1, 'BatchNorm computes unbiased standard-deviation, which requires size > 1.'
+        mean = sum_ / size
+        sumvar = ssum - sum_ * mean
+        unbias_var = sumvar / (size - 1)
+        bias_var = sumvar / size
+
+        self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.data
+        self.running_var = (1 - self.momentum) * self.running_var + self.momentum * unbias_var.data
+
+        return mean, bias_var.clamp(self.eps) ** -0.5
+
+
+class SynchronizedBatchNorm1d(_SynchronizedBatchNorm):
+    r"""Applies Synchronized Batch Normalization over a 2d or 3d input that is seen as a
+    mini-batch.
+    .. math::
+        y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta
+    This module differs from the built-in PyTorch BatchNorm1d as the mean and
+    standard-deviation are reduced across all devices during training.
+    For example, when one uses `nn.DataParallel` to wrap the network during
+    training, PyTorch's implementation normalize the tensor on each device using
+    the statistics only on that device, which accelerated the computation and
+    is also easy to implement, but the statistics might be inaccurate.
+    Instead, in this synchronized version, the statistics will be computed
+    over all training samples distributed on multiple devices.
+
+    Note that, for one-GPU or CPU-only case, this module behaves exactly same
+    as the built-in PyTorch implementation.
+    The mean and standard-deviation are calculated per-dimension over
+    the mini-batches and gamma and beta are learnable parameter vectors
+    of size C (where C is the input size).
+    During training, this layer keeps a running estimate of its computed mean
+    and variance. The running sum is kept with a default momentum of 0.1.
+    During evaluation, this running mean/variance is used for normalization.
+    Because the BatchNorm is done over the `C` dimension, computing statistics
+    on `(N, L)` slices, it's common terminology to call this Temporal BatchNorm
+    Args:
+        num_features: num_features from an expected input of size
+            `batch_size x num_features [x width]`
+        eps: a value added to the denominator for numerical stability.
+            Default: 1e-5
+        momentum: the value used for the running_mean and running_var
+            computation. Default: 0.1
+        affine: a boolean value that when set to ``True``, gives the layer learnable
+            affine parameters. Default: ``True``
+    Shape:
+        - Input: :math:`(N, C)` or :math:`(N, C, L)`
+        - Output: :math:`(N, C)` or :math:`(N, C, L)` (same shape as input)
+    Examples:
+        >>> # With Learnable Parameters
+        >>> m = SynchronizedBatchNorm1d(100)
+        >>> # Without Learnable Parameters
+        >>> m = SynchronizedBatchNorm1d(100, affine=False)
+        >>> input = torch.autograd.Variable(torch.randn(20, 100))
+        >>> output = m(input)
+    """
+
+    def _check_input_dim(self, input):
+        if input.dim() != 2 and input.dim() != 3:
+            raise ValueError('expected 2D or 3D input (got {}D input)'
+                             .format(input.dim()))
+        super(SynchronizedBatchNorm1d, self)._check_input_dim(input)
+
+
+class SynchronizedBatchNorm2d(_SynchronizedBatchNorm):
+    r"""Applies Batch Normalization over a 4d input that is seen as a mini-batch
+    of 3d inputs
+    .. math::
+        y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta
+    This module differs from the built-in PyTorch BatchNorm2d as the mean and
+    standard-deviation are reduced across all devices during training.
+    For example, when one uses `nn.DataParallel` to wrap the network during
+    training, PyTorch's implementation normalize the tensor on each device using
+    the statistics only on that device, which accelerated the computation and
+    is also easy to implement, but the statistics might be inaccurate.
+    Instead, in this synchronized version, the statistics will be computed
+    over all training samples distributed on multiple devices.
+
+    Note that, for one-GPU or CPU-only case, this module behaves exactly same
+    as the built-in PyTorch implementation.
+    The mean and standard-deviation are calculated per-dimension over
+    the mini-batches and gamma and beta are learnable parameter vectors
+    of size C (where C is the input size).
+    During training, this layer keeps a running estimate of its computed mean
+    and variance. The running sum is kept with a default momentum of 0.1.
+    During evaluation, this running mean/variance is used for normalization.
+    Because the BatchNorm is done over the `C` dimension, computing statistics
+    on `(N, H, W)` slices, it's common terminology to call this Spatial BatchNorm
+    Args:
+        num_features: num_features from an expected input of
+            size batch_size x num_features x height x width
+        eps: a value added to the denominator for numerical stability.
+            Default: 1e-5
+        momentum: the value used for the running_mean and running_var
+            computation. Default: 0.1
+        affine: a boolean value that when set to ``True``, gives the layer learnable
+            affine parameters. Default: ``True``
+    Shape:
+        - Input: :math:`(N, C, H, W)`
+        - Output: :math:`(N, C, H, W)` (same shape as input)
+    Examples:
+        >>> # With Learnable Parameters
+        >>> m = SynchronizedBatchNorm2d(100)
+        >>> # Without Learnable Parameters
+        >>> m = SynchronizedBatchNorm2d(100, affine=False)
+        >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45))
+        >>> output = m(input)
+    """
+
+    def _check_input_dim(self, input):
+        if input.dim() != 4:
+            raise ValueError('expected 4D input (got {}D input)'
+                             .format(input.dim()))
+        super(SynchronizedBatchNorm2d, self)._check_input_dim(input)
+
+
+class SynchronizedBatchNorm3d(_SynchronizedBatchNorm):
+    r"""Applies Batch Normalization over a 5d input that is seen as a mini-batch
+    of 4d inputs
+    .. math::
+        y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta
+    This module differs from the built-in PyTorch BatchNorm3d as the mean and
+    standard-deviation are reduced across all devices during training.
+    For example, when one uses `nn.DataParallel` to wrap the network during
+    training, PyTorch's implementation normalize the tensor on each device using
+    the statistics only on that device, which accelerated the computation and
+    is also easy to implement, but the statistics might be inaccurate.
+    Instead, in this synchronized version, the statistics will be computed
+    over all training samples distributed on multiple devices.
+
+    Note that, for one-GPU or CPU-only case, this module behaves exactly same
+    as the built-in PyTorch implementation.
+    The mean and standard-deviation are calculated per-dimension over
+    the mini-batches and gamma and beta are learnable parameter vectors
+    of size C (where C is the input size).
+    During training, this layer keeps a running estimate of its computed mean
+    and variance. The running sum is kept with a default momentum of 0.1.
+    During evaluation, this running mean/variance is used for normalization.
+    Because the BatchNorm is done over the `C` dimension, computing statistics
+    on `(N, D, H, W)` slices, it's common terminology to call this Volumetric BatchNorm
+    or Spatio-temporal BatchNorm
+    Args:
+        num_features: num_features from an expected input of
+            size batch_size x num_features x depth x height x width
+        eps: a value added to the denominator for numerical stability.
+            Default: 1e-5
+        momentum: the value used for the running_mean and running_var
+            computation. Default: 0.1
+        affine: a boolean value that when set to ``True``, gives the layer learnable
+            affine parameters. Default: ``True``
+    Shape:
+        - Input: :math:`(N, C, D, H, W)`
+        - Output: :math:`(N, C, D, H, W)` (same shape as input)
+    Examples:
+        >>> # With Learnable Parameters
+        >>> m = SynchronizedBatchNorm3d(100)
+        >>> # Without Learnable Parameters
+        >>> m = SynchronizedBatchNorm3d(100, affine=False)
+        >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45, 10))
+        >>> output = m(input)
+    """
+
+    def _check_input_dim(self, input):
+        if input.dim() != 5:
+            raise ValueError('expected 5D input (got {}D input)'
+                             .format(input.dim()))
+        super(SynchronizedBatchNorm3d, self)._check_input_dim(input)

core/sync_batchnorm/comm.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+# File   : comm.py
+# Author : Jiayuan Mao
+# Email  : [email protected]
+# Date   : 27/01/2018
+#
+# This file is part of Synchronized-BatchNorm-PyTorch.
+# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
+# Distributed under MIT License.
+
+import queue
+import collections
+import threading
+
+__all__ = ['FutureResult', 'SlavePipe', 'SyncMaster']
+
+
+class FutureResult(object):
+    """A thread-safe future implementation. Used only as one-to-one pipe."""
+
+    def __init__(self):
+        self._result = None
+        self._lock = threading.Lock()
+        self._cond = threading.Condition(self._lock)
+
+    def put(self, result):
+        with self._lock:
+            assert self._result is None, 'Previous result has\'t been fetched.'
+            self._result = result
+            self._cond.notify()
+
+    def get(self):
+        with self._lock:
+            if self._result is None:
+                self._cond.wait()
+
+            res = self._result
+            self._result = None
+            return res
+
+
+_MasterRegistry = collections.namedtuple('MasterRegistry', ['result'])
+_SlavePipeBase = collections.namedtuple('_SlavePipeBase', ['identifier', 'queue', 'result'])
+
+
+class SlavePipe(_SlavePipeBase):
+    """Pipe for master-slave communication."""
+
+    def run_slave(self, msg):
+        self.queue.put((self.identifier, msg))
+        ret = self.result.get()
+        self.queue.put(True)
+        return ret
+
+
+class SyncMaster(object):
+    """An abstract `SyncMaster` object.
+    - During the replication, as the data parallel will trigger an callback of each module, all slave devices should
+    call `register(id)` and obtain an `SlavePipe` to communicate with the master.
+    - During the forward pass, master device invokes `run_master`, all messages from slave devices will be collected,
+    and passed to a registered callback.
+    - After receiving the messages, the master device should gather the information and determine to message passed
+    back to each slave devices.
+    """
+
+    def __init__(self, master_callback):
+        """
+        Args:
+            master_callback: a callback to be invoked after having collected messages from slave devices.
+        """
+        self._master_callback = master_callback
+        self._queue = queue.Queue()
+        self._registry = collections.OrderedDict()
+        self._activated = False
+
+    def __getstate__(self):
+        return {'master_callback': self._master_callback}
+
+    def __setstate__(self, state):
+        self.__init__(state['master_callback'])
+
+    def register_slave(self, identifier):
+        """
+        Register an slave device.
+        Args:
+            identifier: an identifier, usually is the device id.
+        Returns: a `SlavePipe` object which can be used to communicate with the master device.
+        """
+        if self._activated:
+            assert self._queue.empty(), 'Queue is not clean before next initialization.'
+            self._activated = False
+            self._registry.clear()
+        future = FutureResult()
+        self._registry[identifier] = _MasterRegistry(future)
+        return SlavePipe(identifier, self._queue, future)
+
+    def run_master(self, master_msg):
+        """
+        Main entry for the master device in each forward pass.
+        The messages were first collected from each devices (including the master device), and then
+        an callback will be invoked to compute the message to be sent back to each devices
+        (including the master device).
+        Args:
+            master_msg: the message that the master want to send to itself. This will be placed as the first
+            message when calling `master_callback`. For detailed usage, see `_SynchronizedBatchNorm` for an example.
+        Returns: the message to be sent back to the master device.
+        """
+        self._activated = True
+
+        intermediates = [(0, master_msg)]
+        for i in range(self.nr_slaves):
+            intermediates.append(self._queue.get())
+
+        results = self._master_callback(intermediates)
+        assert results[0][0] == 0, 'The first result should belongs to the master.'
+
+        for i, res in results:
+            if i == 0:
+                continue
+            self._registry[i].result.put(res)
+
+        for i in range(self.nr_slaves):
+            assert self._queue.get() is True
+
+        return results[0][1]
+
+    @property
+    def nr_slaves(self):
+        return len(self._registry)

core/sync_batchnorm/replicate.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+# File   : replicate.py
+# Author : Jiayuan Mao
+# Email  : [email protected]
+# Date   : 27/01/2018
+#
+# This file is part of Synchronized-BatchNorm-PyTorch.
+# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
+# Distributed under MIT License.
+
+import functools
+
+from torch.nn.parallel.data_parallel import DataParallel
+
+__all__ = [
+    'CallbackContext',
+    'execute_replication_callbacks',
+    'DataParallelWithCallback',
+    'patch_replication_callback'
+]
+
+
+class CallbackContext(object):
+    pass
+
+
+def execute_replication_callbacks(modules):
+    """
+    Execute an replication callback `__data_parallel_replicate__` on each module created by original replication.
+    The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)`
+    Note that, as all modules are isomorphism, we assign each sub-module with a context
+    (shared among multiple copies of this module on different devices).
+    Through this context, different copies can share some information.
+    We guarantee that the callback on the master copy (the first copy) will be called ahead of calling the callback
+    of any slave copies.
+    """
+    master_copy = modules[0]
+    nr_modules = len(list(master_copy.modules()))
+    ctxs = [CallbackContext() for _ in range(nr_modules)]
+
+    for i, module in enumerate(modules):
+        for j, m in enumerate(module.modules()):
+            if hasattr(m, '__data_parallel_replicate__'):
+                m.__data_parallel_replicate__(ctxs[j], i)
+
+
+class DataParallelWithCallback(DataParallel):
+    """
+    Data Parallel with a replication callback.
+    An replication callback `__data_parallel_replicate__` of each module will be invoked after being created by
+    original `replicate` function.
+    The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)`
+    Examples:
+        > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False)
+        > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1])
+        # sync_bn.__data_parallel_replicate__ will be invoked.
+    """
+
+    def replicate(self, module, device_ids):
+        modules = super(DataParallelWithCallback, self).replicate(module, device_ids)
+        execute_replication_callbacks(modules)
+        return modules
+
+
+def patch_replication_callback(data_parallel):
+    """
+    Monkey-patch an existing `DataParallel` object. Add the replication callback.
+    Useful when you have customized `DataParallel` implementation.
+    Examples:
+        > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False)
+        > sync_bn = DataParallel(sync_bn, device_ids=[0, 1])
+        > patch_replication_callback(sync_bn)
+        # this is equivalent to
+        > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False)
+        > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1])
+    """
+
+    assert isinstance(data_parallel, DataParallel)
+
+    old_replicate = data_parallel.replicate
+
+    @functools.wraps(old_replicate)
+    def new_replicate(module, device_ids):
+        modules = old_replicate(module, device_ids)
+        execute_replication_callbacks(modules)
+        return modules
+
+    data_parallel.replicate = new_replicate

core/sync_batchnorm/unittest.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# File   : unittest.py
+# Author : Jiayuan Mao
+# Email  : [email protected]
+# Date   : 27/01/2018
+#
+# This file is part of Synchronized-BatchNorm-PyTorch.
+# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
+# Distributed under MIT License.
+
+import unittest
+
+import numpy as np
+from torch.autograd import Variable
+
+
+def as_numpy(v):
+    if isinstance(v, Variable):
+        v = v.data
+    return v.cpu().numpy()
+
+
+class TorchTestCase(unittest.TestCase):
+    def assertTensorClose(self, a, b, atol=1e-3, rtol=1e-3):
+        npa, npb = as_numpy(a), as_numpy(b)
+        self.assertTrue(
+                np.allclose(npa, npb, atol=atol),
+                'Tensor close check failed\n{}\n{}\nadiff={}, rdiff={}'.format(a, b, np.abs(npa - npb).max(), np.abs((npa - npb) / np.fmax(npa, 1e-5)).max())
+        )

data/VOC_2012.json

@@ -0,0 +1,198 @@
+{
+	"train": {
+		"aeroplane": 837,
+		"person": 9734,
+		"tvmonitor": 790,
+		"dog": 1418,
+		"chair": 2794,
+		"bird": 1128,
+		"bottle": 1396,
+		"boat": 940,
+		"diningtable": 715,
+		"train": 609,
+		"motorbike": 689,
+		"horse": 696,
+		"cow": 633,
+		"bicycle": 712,
+		"car": 2235,
+		"cat": 1141,
+		"sofa": 732,
+		"bus": 569,
+		"pottedplant": 1024,
+		"sheep": 931
+	},
+	"validation": {
+		"aeroplane": 112,
+		"train": 93,
+		"boat": 108,
+		"bicycle": 103,
+		"person": 866,
+		"sheep": 153,
+		"tvmonitor": 98,
+		"horse": 104,
+		"bottle": 163,
+		"sofa": 106,
+		"chair": 245,
+		"cow": 132,
+		"car": 249,
+		"pottedplant": 171,
+		"bus": 116,
+		"diningtable": 82,
+		"dog": 150,
+		"bird": 140,
+		"cat": 132,
+		"motorbike": 103
+	},
+	"classes": 20,
+	"class_names": [
+		"aeroplane",
+		"bicycle",
+		"bird",
+		"boat",
+		"bottle",
+		"bus",
+		"car",
+		"cat",
+		"chair",
+		"cow",
+		"diningtable",
+		"dog",
+		"horse",
+		"motorbike",
+		"person",
+		"pottedplant",
+		"sheep",
+		"sofa",
+		"train",
+		"tvmonitor"
+	],
+	"class_dic": {
+		"aeroplane": 0,
+		"bicycle": 1,
+		"bird": 2,
+		"boat": 3,
+		"bottle": 4,
+		"bus": 5,
+		"car": 6,
+		"cat": 7,
+		"chair": 8,
+		"cow": 9,
+		"diningtable": 10,
+		"dog": 11,
+		"horse": 12,
+		"motorbike": 13,
+		"person": 14,
+		"pottedplant": 15,
+		"sheep": 16,
+		"sofa": 17,
+		"train": 18,
+		"tvmonitor": 19
+	},
+	"color_dict": {
+		"background": [
+			0,
+			0,
+			0
+		],
+		"aeroplane": [
+			128,
+			0,
+			0
+		],
+		"bicycle": [
+			0,
+			128,
+			0
+		],
+		"bird": [
+			128,
+			128,
+			0
+		],
+		"boat": [
+			0,
+			0,
+			128
+		],
+		"bottle": [
+			128,
+			0,
+			128
+		],
+		"bus": [
+			0,
+			128,
+			128
+		],
+		"car": [
+			128,
+			128,
+			128
+		],
+		"cat": [
+			64,
+			0,
+			0
+		],
+		"chair": [
+			192,
+			0,
+			0
+		],
+		"cow": [
+			64,
+			128,
+			0
+		],
+		"diningtable": [
+			192,
+			128,
+			0
+		],
+		"dog": [
+			64,
+			0,
+			128
+		],
+		"horse": [
+			192,
+			0,
+			128
+		],
+		"motorbike": [
+			64,
+			128,
+			128
+		],
+		"person": [
+			192,
+			128,
+			128
+		],
+		"pottedplant": [
+			0,
+			64,
+			0
+		],
+		"sheep": [
+			128,
+			64,
+			0
+		],
+		"sofa": [
+			0,
+			192,
+			0
+		],
+		"train": [
+			128,
+			192,
+			0
+		],
+		"tvmonitor": [
+			0,
+			64,
+			128
+		]
+	}
+}

data/test.txt

@@ -0,0 +1,1456 @@
+2008_000006
+2008_000011
+2008_000012
+2008_000018
+2008_000024
+2008_000030
+2008_000031
+2008_000046
+2008_000047
+2008_000048
+2008_000057
+2008_000058
+2008_000068
+2008_000072
+2008_000079
+2008_000081
+2008_000083
+2008_000088
+2008_000094
+2008_000101
+2008_000104
+2008_000106
+2008_000108
+2008_000110
+2008_000111
+2008_000126
+2008_000127
+2008_000129
+2008_000130
+2008_000135
+2008_000150
+2008_000152
+2008_000156
+2008_000159
+2008_000160
+2008_000161
+2008_000166
+2008_000167
+2008_000168
+2008_000169
+2008_000171
+2008_000175
+2008_000178
+2008_000186
+2008_000198
+2008_000206
+2008_000208
+2008_000209
+2008_000211
+2008_000220
+2008_000224
+2008_000230
+2008_000240
+2008_000248
+2008_000249
+2008_000250
+2008_000256
+2008_000279
+2008_000282
+2008_000285
+2008_000286
+2008_000296
+2008_000300
+2008_000322
+2008_000324
+2008_000337
+2008_000366
+2008_000369
+2008_000377
+2008_000384
+2008_000390
+2008_000404
+2008_000411
+2008_000434
+2008_000440
+2008_000460
+2008_000467
+2008_000478
+2008_000485
+2008_000487
+2008_000490
+2008_000503
+2008_000504
+2008_000507
+2008_000513
+2008_000523
+2008_000529
+2008_000556
+2008_000565
+2008_000580
+2008_000590
+2008_000596
+2008_000597
+2008_000600
+2008_000603
+2008_000604
+2008_000612
+2008_000617
+2008_000621
+2008_000627
+2008_000633
+2008_000643
+2008_000644
+2008_000649
+2008_000651
+2008_000664
+2008_000665
+2008_000680
+2008_000681
+2008_000684
+2008_000685
+2008_000688
+2008_000693
+2008_000698
+2008_000707
+2008_000709
+2008_000712
+2008_000747
+2008_000751
+2008_000754
+2008_000762
+2008_000767
+2008_000768
+2008_000773
+2008_000774
+2008_000779
+2008_000797
+2008_000813
+2008_000816
+2008_000846
+2008_000866
+2008_000871
+2008_000872
+2008_000891
+2008_000892
+2008_000894
+2008_000896
+2008_000898
+2008_000909
+2008_000913
+2008_000920
+2008_000933
+2008_000935
+2008_000937
+2008_000938
+2008_000954
+2008_000958
+2008_000963
+2008_000967
+2008_000974
+2008_000986
+2008_000994
+2008_000995
+2008_001008
+2008_001010
+2008_001014
+2008_001016
+2008_001025
+2008_001029
+2008_001037
+2008_001059
+2008_001061
+2008_001072
+2008_001124
+2008_001126
+2008_001131
+2008_001138
+2008_001144
+2008_001151
+2008_001156
+2008_001179
+2008_001181
+2008_001184
+2008_001186
+2008_001197
+2008_001207
+2008_001212
+2008_001233
+2008_001234
+2008_001258
+2008_001268
+2008_001279
+2008_001281
+2008_001288
+2008_001291
+2008_001298
+2008_001309
+2008_001315
+2008_001316
+2008_001319
+2008_001327
+2008_001328
+2008_001332
+2008_001341
+2008_001347
+2008_001355
+2008_001378
+2008_001386
+2008_001400
+2008_001409
+2008_001411
+2008_001416
+2008_001418
+2008_001435
+2008_001459
+2008_001469
+2008_001474
+2008_001477
+2008_001483
+2008_001484
+2008_001485
+2008_001496
+2008_001507
+2008_001511
+2008_001519
+2008_001557
+2008_001567
+2008_001570
+2008_001571
+2008_001572
+2008_001579
+2008_001587
+2008_001608
+2008_001611
+2008_001614
+2008_001621
+2008_001639
+2008_001658
+2008_001678
+2008_001700
+2008_001713
+2008_001720
+2008_001755
+2008_001779
+2008_001785
+2008_001793
+2008_001794
+2008_001803
+2008_001818
+2008_001848
+2008_001855
+2008_001857
+2008_001861
+2008_001875
+2008_001878
+2008_001886
+2008_001897
+2008_001916
+2008_001925
+2008_001949
+2008_001953
+2008_001972
+2008_001999
+2008_002027
+2008_002040
+2008_002057
+2008_002070
+2008_002075
+2008_002095
+2008_002104
+2008_002105
+2008_002106
+2008_002136
+2008_002137
+2008_002147
+2008_002149
+2008_002163
+2008_002173
+2008_002174
+2008_002184
+2008_002186
+2008_002188
+2008_002190
+2008_002203
+2008_002211
+2008_002217
+2008_002228
+2008_002233
+2008_002246
+2008_002257
+2008_002261
+2008_002285
+2008_002287
+2008_002295
+2008_002303
+2008_002306
+2008_002309
+2008_002310
+2008_002318
+2008_002320
+2008_002332
+2008_002337
+2008_002345
+2008_002348
+2008_002352
+2008_002360
+2008_002381
+2008_002387
+2008_002388
+2008_002393
+2008_002406
+2008_002440
+2008_002455
+2008_002460
+2008_002462
+2008_002480
+2008_002518
+2008_002525
+2008_002535
+2008_002544
+2008_002553
+2008_002569
+2008_002572
+2008_002587
+2008_002635
+2008_002655
+2008_002695
+2008_002702
+2008_002706
+2008_002707
+2008_002722
+2008_002745
+2008_002757
+2008_002779
+2008_002805
+2008_002871
+2008_002895
+2008_002905
+2008_002923
+2008_002927
+2008_002939
+2008_002941
+2008_002962
+2008_002975
+2008_003000
+2008_003031
+2008_003038
+2008_003042
+2008_003069
+2008_003070
+2008_003115
+2008_003116
+2008_003130
+2008_003137
+2008_003138
+2008_003139
+2008_003165
+2008_003171
+2008_003176
+2008_003192
+2008_003194
+2008_003195
+2008_003198
+2008_003227
+2008_003247
+2008_003262
+2008_003298
+2008_003299
+2008_003307
+2008_003337
+2008_003353
+2008_003355
+2008_003363
+2008_003383
+2008_003389
+2008_003392
+2008_003399
+2008_003436
+2008_003457
+2008_003465
+2008_003481
+2008_003539
+2008_003548
+2008_003550
+2008_003567
+2008_003568
+2008_003606
+2008_003615
+2008_003654
+2008_003670
+2008_003700
+2008_003705
+2008_003727
+2008_003731
+2008_003734
+2008_003760
+2008_003804
+2008_003807
+2008_003810
+2008_003822
+2008_003833
+2008_003877
+2008_003879
+2008_003895
+2008_003901
+2008_003903
+2008_003911
+2008_003919
+2008_003927
+2008_003937
+2008_003946
+2008_003950
+2008_003955
+2008_003981
+2008_003991
+2008_004009
+2008_004039
+2008_004052
+2008_004063
+2008_004070
+2008_004078
+2008_004104
+2008_004139
+2008_004177
+2008_004181
+2008_004200
+2008_004219
+2008_004236
+2008_004250
+2008_004266
+2008_004299
+2008_004320
+2008_004334
+2008_004343
+2008_004349
+2008_004366
+2008_004386
+2008_004401
+2008_004423
+2008_004448
+2008_004481
+2008_004516
+2008_004536
+2008_004582
+2008_004609
+2008_004638
+2008_004642
+2008_004644
+2008_004669
+2008_004673
+2008_004691
+2008_004693
+2008_004709
+2008_004715
+2008_004757
+2008_004775
+2008_004782
+2008_004785
+2008_004798
+2008_004848
+2008_004861
+2008_004870
+2008_004877
+2008_004884
+2008_004891
+2008_004901
+2008_004919
+2008_005058
+2008_005069
+2008_005086
+2008_005087
+2008_005112
+2008_005113
+2008_005118
+2008_005128
+2008_005129
+2008_005153
+2008_005161
+2008_005162
+2008_005165
+2008_005187
+2008_005227
+2008_005308
+2008_005318
+2008_005320
+2008_005351
+2008_005372
+2008_005383
+2008_005391
+2008_005407
+2008_005420
+2008_005440
+2008_005487
+2008_005493
+2008_005520
+2008_005551
+2008_005556
+2008_005576
+2008_005578
+2008_005594
+2008_005619
+2008_005629
+2008_005644
+2008_005645
+2008_005651
+2008_005661
+2008_005662
+2008_005667
+2008_005694
+2008_005697
+2008_005709
+2008_005710
+2008_005733
+2008_005749
+2008_005753
+2008_005771
+2008_005781
+2008_005793
+2008_005802
+2008_005833
+2008_005844
+2008_005908
+2008_005931
+2008_005952
+2008_006016
+2008_006030
+2008_006033
+2008_006054
+2008_006073
+2008_006091
+2008_006142
+2008_006150
+2008_006206
+2008_006217
+2008_006264
+2008_006283
+2008_006308
+2008_006313
+2008_006333
+2008_006343
+2008_006381
+2008_006391
+2008_006423
+2008_006428
+2008_006440
+2008_006444
+2008_006473
+2008_006505
+2008_006531
+2008_006560
+2008_006571
+2008_006582
+2008_006594
+2008_006601
+2008_006633
+2008_006653
+2008_006678
+2008_006755
+2008_006772
+2008_006788
+2008_006799
+2008_006809
+2008_006838
+2008_006845
+2008_006852
+2008_006894
+2008_006905
+2008_006947
+2008_006983
+2008_007049
+2008_007065
+2008_007068
+2008_007111
+2008_007148
+2008_007159
+2008_007193
+2008_007228
+2008_007235
+2008_007249
+2008_007255
+2008_007268
+2008_007275
+2008_007292
+2008_007299
+2008_007306
+2008_007316
+2008_007400
+2008_007401
+2008_007419
+2008_007437
+2008_007483
+2008_007487
+2008_007520
+2008_007551
+2008_007603
+2008_007616
+2008_007654
+2008_007663
+2008_007708
+2008_007795
+2008_007801
+2008_007859
+2008_007903
+2008_007920
+2008_007926
+2008_008014
+2008_008017
+2008_008060
+2008_008077
+2008_008107
+2008_008108
+2008_008119
+2008_008126
+2008_008133
+2008_008144
+2008_008216
+2008_008244
+2008_008248
+2008_008250
+2008_008260
+2008_008277
+2008_008280
+2008_008290
+2008_008304
+2008_008340
+2008_008371
+2008_008390
+2008_008397
+2008_008409
+2008_008412
+2008_008419
+2008_008454
+2008_008491
+2008_008498
+2008_008565
+2008_008599
+2008_008603
+2008_008631
+2008_008634
+2008_008640
+2008_008646
+2008_008660
+2008_008663
+2008_008664
+2008_008709
+2008_008720
+2008_008747
+2008_008768
+2009_000004
+2009_000019
+2009_000024
+2009_000025
+2009_000053
+2009_000076
+2009_000107
+2009_000110
+2009_000115
+2009_000117
+2009_000175
+2009_000220
+2009_000259
+2009_000275
+2009_000314
+2009_000368
+2009_000373
+2009_000384
+2009_000388
+2009_000423
+2009_000433
+2009_000434
+2009_000458
+2009_000475
+2009_000481
+2009_000495
+2009_000514
+2009_000555
+2009_000556
+2009_000561
+2009_000571
+2009_000581
+2009_000605
+2009_000609
+2009_000644
+2009_000654
+2009_000671
+2009_000733
+2009_000740
+2009_000766
+2009_000775
+2009_000776
+2009_000795
+2009_000850
+2009_000881
+2009_000900
+2009_000914
+2009_000941
+2009_000977
+2009_000984
+2009_000986
+2009_001005
+2009_001015
+2009_001058
+2009_001072
+2009_001087
+2009_001092
+2009_001109
+2009_001114
+2009_001115
+2009_001141
+2009_001174
+2009_001175
+2009_001182
+2009_001222
+2009_001228
+2009_001246
+2009_001262
+2009_001274
+2009_001284
+2009_001297
+2009_001331
+2009_001336
+2009_001337
+2009_001379
+2009_001392
+2009_001451
+2009_001485
+2009_001488
+2009_001497
+2009_001504
+2009_001506
+2009_001573
+2009_001576
+2009_001603
+2009_001613
+2009_001652
+2009_001661
+2009_001668
+2009_001680
+2009_001688
+2009_001697
+2009_001729
+2009_001771
+2009_001785
+2009_001793
+2009_001814
+2009_001866
+2009_001872
+2009_001880
+2009_001883
+2009_001891
+2009_001913
+2009_001938
+2009_001946
+2009_001953
+2009_001969
+2009_001978
+2009_001995
+2009_002007
+2009_002036
+2009_002041
+2009_002049
+2009_002051
+2009_002062
+2009_002063
+2009_002067
+2009_002085
+2009_002092
+2009_002114
+2009_002115
+2009_002142
+2009_002148
+2009_002157
+2009_002181
+2009_002220
+2009_002284
+2009_002287
+2009_002300
+2009_002310
+2009_002315
+2009_002334
+2009_002337
+2009_002354
+2009_002357
+2009_002411
+2009_002426
+2009_002458
+2009_002459
+2009_002461
+2009_002466
+2009_002481
+2009_002483
+2009_002503
+2009_002581
+2009_002583
+2009_002589
+2009_002600
+2009_002601
+2009_002602
+2009_002641
+2009_002646
+2009_002656
+2009_002666
+2009_002720
+2009_002767
+2009_002768
+2009_002794
+2009_002821
+2009_002825
+2009_002839
+2009_002840
+2009_002859
+2009_002860
+2009_002881
+2009_002889
+2009_002892
+2009_002895
+2009_002896
+2009_002900
+2009_002924
+2009_002966
+2009_002973
+2009_002981
+2009_003004
+2009_003021
+2009_003028
+2009_003037
+2009_003038
+2009_003055
+2009_003085
+2009_003100
+2009_003106
+2009_003117
+2009_003139
+2009_003170
+2009_003179
+2009_003184
+2009_003186
+2009_003190
+2009_003221
+2009_003236
+2009_003242
+2009_003244
+2009_003260
+2009_003264
+2009_003274
+2009_003283
+2009_003296
+2009_003332
+2009_003341
+2009_003354
+2009_003370
+2009_003371
+2009_003374
+2009_003391
+2009_003393
+2009_003404
+2009_003405
+2009_003414
+2009_003428
+2009_003470
+2009_003474
+2009_003532
+2009_003536
+2009_003578
+2009_003580
+2009_003620
+2009_003621
+2009_003680
+2009_003699
+2009_003727
+2009_003737
+2009_003780
+2009_003811
+2009_003824
+2009_003831
+2009_003844
+2009_003850
+2009_003851
+2009_003864
+2009_003868
+2009_003869
+2009_003893
+2009_003909
+2009_003924
+2009_003925
+2009_003960
+2009_003979
+2009_003990
+2009_003997
+2009_004006
+2009_004010
+2009_004066
+2009_004077
+2009_004081
+2009_004097
+2009_004098
+2009_004136
+2009_004216
+2009_004220
+2009_004266
+2009_004269
+2009_004286
+2009_004296
+2009_004321
+2009_004342
+2009_004343
+2009_004344
+2009_004385
+2009_004408
+2009_004420
+2009_004441
+2009_004447
+2009_004461
+2009_004467
+2009_004485
+2009_004488
+2009_004516
+2009_004521
+2009_004544
+2009_004596
+2009_004613
+2009_004615
+2009_004618
+2009_004621
+2009_004646
+2009_004659
+2009_004663
+2009_004666
+2009_004691
+2009_004715
+2009_004726
+2009_004753
+2009_004776
+2009_004811
+2009_004814
+2009_004818
+2009_004835
+2009_004863
+2009_004894
+2009_004909
+2009_004928
+2009_004937
+2009_004954
+2009_004966
+2009_004970
+2009_004976
+2009_005004
+2009_005011
+2009_005053
+2009_005072
+2009_005115
+2009_005146
+2009_005151
+2009_005164
+2009_005179
+2009_005224
+2009_005243
+2009_005249
+2009_005252
+2009_005254
+2009_005258
+2009_005264
+2009_005266
+2009_005276
+2009_005290
+2009_005295
+2010_000004
+2010_000005
+2010_000006
+2010_000032
+2010_000062
+2010_000093
+2010_000094
+2010_000161
+2010_000176
+2010_000223
+2010_000226
+2010_000236
+2010_000239
+2010_000287
+2010_000300
+2010_000301
+2010_000328
+2010_000378
+2010_000405
+2010_000407
+2010_000472
+2010_000479
+2010_000491
+2010_000533
+2010_000535
+2010_000542
+2010_000554
+2010_000580
+2010_000594
+2010_000596
+2010_000599
+2010_000606
+2010_000615
+2010_000654
+2010_000659
+2010_000693
+2010_000698
+2010_000730
+2010_000734
+2010_000741
+2010_000755
+2010_000768
+2010_000794
+2010_000813
+2010_000817
+2010_000834
+2010_000839
+2010_000848
+2010_000881
+2010_000888
+2010_000900
+2010_000903
+2010_000924
+2010_000946
+2010_000953
+2010_000957
+2010_000967
+2010_000992
+2010_000998
+2010_001053
+2010_001067
+2010_001114
+2010_001132
+2010_001138
+2010_001169
+2010_001171
+2010_001228
+2010_001260
+2010_001268
+2010_001280
+2010_001298
+2010_001302
+2010_001308
+2010_001324
+2010_001332
+2010_001335
+2010_001345
+2010_001346
+2010_001349
+2010_001373
+2010_001381
+2010_001392
+2010_001396
+2010_001420
+2010_001500
+2010_001506
+2010_001521
+2010_001532
+2010_001558
+2010_001598
+2010_001611
+2010_001631
+2010_001639
+2010_001651
+2010_001663
+2010_001664
+2010_001728
+2010_001778
+2010_001861
+2010_001874
+2010_001900
+2010_001905
+2010_001969
+2010_002008
+2010_002014
+2010_002049
+2010_002052
+2010_002091
+2010_002115
+2010_002119
+2010_002134
+2010_002156
+2010_002160
+2010_002186
+2010_002210
+2010_002241
+2010_002252
+2010_002258
+2010_002262
+2010_002273
+2010_002290
+2010_002292
+2010_002347
+2010_002358
+2010_002360
+2010_002367
+2010_002416
+2010_002451
+2010_002481
+2010_002490
+2010_002495
+2010_002588
+2010_002607
+2010_002609
+2010_002610
+2010_002641
+2010_002685
+2010_002699
+2010_002719
+2010_002735
+2010_002751
+2010_002804
+2010_002835
+2010_002852
+2010_002885
+2010_002889
+2010_002904
+2010_002908
+2010_002916
+2010_002974
+2010_002977
+2010_003005
+2010_003021
+2010_003030
+2010_003038
+2010_003046
+2010_003052
+2010_003089
+2010_003110
+2010_003118
+2010_003171
+2010_003217
+2010_003221
+2010_003228
+2010_003243
+2010_003271
+2010_003295
+2010_003306
+2010_003324
+2010_003363
+2010_003382
+2010_003388
+2010_003389
+2010_003392
+2010_003430
+2010_003442
+2010_003459
+2010_003485
+2010_003486
+2010_003500
+2010_003523
+2010_003542
+2010_003552
+2010_003570
+2010_003572
+2010_003586
+2010_003615
+2010_003623
+2010_003657
+2010_003666
+2010_003705
+2010_003710
+2010_003720
+2010_003733
+2010_003750
+2010_003767
+2010_003802
+2010_003809
+2010_003830
+2010_003832
+2010_003836
+2010_003838
+2010_003850
+2010_003867
+2010_003882
+2010_003909
+2010_003922
+2010_003923
+2010_003978
+2010_003989
+2010_003990
+2010_004000
+2010_004003
+2010_004068
+2010_004076
+2010_004117
+2010_004136
+2010_004142
+2010_004195
+2010_004200
+2010_004202
+2010_004232
+2010_004261
+2010_004266
+2010_004273
+2010_004305
+2010_004403
+2010_004433
+2010_004434
+2010_004435
+2010_004438
+2010_004442
+2010_004473
+2010_004482
+2010_004487
+2010_004489
+2010_004512
+2010_004525
+2010_004527
+2010_004532
+2010_004566
+2010_004568
+2010_004579
+2010_004611
+2010_004641
+2010_004688
+2010_004699
+2010_004702
+2010_004716
+2010_004754
+2010_004767
+2010_004776
+2010_004811
+2010_004837
+2010_004839
+2010_004845
+2010_004860
+2010_004867
+2010_004881
+2010_004939
+2010_005001
+2010_005047
+2010_005051
+2010_005091
+2010_005095
+2010_005125
+2010_005140
+2010_005177
+2010_005178
+2010_005194
+2010_005197
+2010_005200
+2010_005205
+2010_005212
+2010_005248
+2010_005294
+2010_005298
+2010_005313
+2010_005324
+2010_005328
+2010_005329
+2010_005380
+2010_005404
+2010_005407
+2010_005411
+2010_005423
+2010_005499
+2010_005509
+2010_005510
+2010_005544
+2010_005549
+2010_005590
+2010_005639
+2010_005699
+2010_005704
+2010_005707
+2010_005711
+2010_005726
+2010_005741
+2010_005765
+2010_005790
+2010_005792
+2010_005797
+2010_005812
+2010_005850
+2010_005861
+2010_005869
+2010_005908
+2010_005915
+2010_005946
+2010_005965
+2010_006044
+2010_006047
+2010_006052
+2010_006081
+2011_000001
+2011_000013
+2011_000014
+2011_000020
+2011_000032
+2011_000042
+2011_000063
+2011_000115
+2011_000120
+2011_000240
+2011_000244
+2011_000254
+2011_000261
+2011_000262
+2011_000271
+2011_000274
+2011_000306
+2011_000311
+2011_000316
+2011_000328
+2011_000351
+2011_000352
+2011_000406
+2011_000414
+2011_000448
+2011_000451
+2011_000470
+2011_000473
+2011_000515
+2011_000537
+2011_000576
+2011_000603
+2011_000616
+2011_000636
+2011_000639
+2011_000654
+2011_000660
+2011_000664
+2011_000667
+2011_000670
+2011_000676
+2011_000721
+2011_000723
+2011_000762
+2011_000766
+2011_000786
+2011_000802
+2011_000810
+2011_000821
+2011_000841
+2011_000844
+2011_000846
+2011_000869
+2011_000890
+2011_000915
+2011_000924
+2011_000937
+2011_000939
+2011_000952
+2011_000968
+2011_000974
+2011_001037
+2011_001072
+2011_001085
+2011_001089
+2011_001090
+2011_001099
+2011_001104
+2011_001112
+2011_001120
+2011_001132
+2011_001151
+2011_001194
+2011_001258
+2011_001274
+2011_001314
+2011_001317
+2011_001321
+2011_001379
+2011_001425
+2011_001431
+2011_001443
+2011_001446
+2011_001452
+2011_001454
+2011_001477
+2011_001509
+2011_001512
+2011_001515
+2011_001528
+2011_001554
+2011_001561
+2011_001580
+2011_001587
+2011_001623
+2011_001648
+2011_001651
+2011_001654
+2011_001684
+2011_001696
+2011_001697
+2011_001760
+2011_001761
+2011_001798
+2011_001807
+2011_001851
+2011_001852
+2011_001853
+2011_001888
+2011_001940
+2011_002014
+2011_002028
+2011_002056
+2011_002061
+2011_002068
+2011_002076
+2011_002090
+2011_002095
+2011_002104
+2011_002136
+2011_002138
+2011_002151
+2011_002153
+2011_002155
+2011_002197
+2011_002198
+2011_002243
+2011_002250
+2011_002257
+2011_002262
+2011_002264
+2011_002296
+2011_002314
+2011_002331
+2011_002333
+2011_002411
+2011_002417
+2011_002425
+2011_002437
+2011_002444
+2011_002445
+2011_002449
+2011_002468
+2011_002469
+2011_002473
+2011_002508
+2011_002523
+2011_002534
+2011_002557
+2011_002564
+2011_002572
+2011_002597
+2011_002622
+2011_002632
+2011_002635
+2011_002643
+2011_002653
+2011_002667
+2011_002681
+2011_002707
+2011_002736
+2011_002759
+2011_002783
+2011_002792
+2011_002799
+2011_002824
+2011_002835
+2011_002866
+2011_002876
+2011_002888
+2011_002894
+2011_002903
+2011_002905
+2011_002986
+2011_003045
+2011_003064
+2011_003070
+2011_003083
+2011_003093
+2011_003096
+2011_003102
+2011_003156
+2011_003170
+2011_003178
+2011_003231

data/train.txt

@@ -0,0 +1,1464 @@
+2007_000032
+2007_000039
+2007_000063
+2007_000068
+2007_000121
+2007_000170
+2007_000241
+2007_000243
+2007_000250
+2007_000256
+2007_000333
+2007_000363
+2007_000364
+2007_000392
+2007_000480
+2007_000504
+2007_000515
+2007_000528
+2007_000549
+2007_000584
+2007_000645
+2007_000648
+2007_000713
+2007_000720
+2007_000733
+2007_000738
+2007_000768
+2007_000793
+2007_000822
+2007_000836
+2007_000876
+2007_000904
+2007_001027
+2007_001073
+2007_001149
+2007_001185
+2007_001225
+2007_001340
+2007_001397
+2007_001416
+2007_001420
+2007_001439
+2007_001487
+2007_001595
+2007_001602
+2007_001609
+2007_001698
+2007_001704
+2007_001709
+2007_001724
+2007_001764
+2007_001825
+2007_001834
+2007_001857
+2007_001872
+2007_001901
+2007_001917
+2007_001960
+2007_002024
+2007_002055
+2007_002088
+2007_002099
+2007_002105
+2007_002107
+2007_002120
+2007_002142
+2007_002198
+2007_002212
+2007_002216
+2007_002227
+2007_002234
+2007_002273
+2007_002281
+2007_002293
+2007_002361
+2007_002368
+2007_002370
+2007_002403
+2007_002462
+2007_002488
+2007_002545
+2007_002611
+2007_002639
+2007_002668
+2007_002669
+2007_002760
+2007_002789
+2007_002845
+2007_002895
+2007_002896
+2007_002914
+2007_002953
+2007_002954
+2007_002967
+2007_003000
+2007_003118
+2007_003178
+2007_003189
+2007_003190
+2007_003191
+2007_003205
+2007_003207
+2007_003251
+2007_003267
+2007_003286
+2007_003330
+2007_003431
+2007_003451
+2007_003525
+2007_003529
+2007_003541
+2007_003565
+2007_003580
+2007_003593
+2007_003604
+2007_003668
+2007_003715
+2007_003778
+2007_003788
+2007_003815
+2007_003876
+2007_003889
+2007_003910
+2007_004003
+2007_004009
+2007_004065
+2007_004081
+2007_004166
+2007_004289
+2007_004291
+2007_004328
+2007_004423
+2007_004459
+2007_004476
+2007_004481
+2007_004500
+2007_004537
+2007_004627
+2007_004663
+2007_004705
+2007_004707
+2007_004768
+2007_004769
+2007_004810
+2007_004830
+2007_004841
+2007_004948
+2007_004951
+2007_004988
+2007_004998
+2007_005043
+2007_005064
+2007_005086
+2007_005124
+2007_005130
+2007_005144
+2007_005210
+2007_005212
+2007_005227
+2007_005248
+2007_005262
+2007_005264
+2007_005266
+2007_005273
+2007_005314
+2007_005360
+2007_005368
+2007_005430
+2007_005647
+2007_005688
+2007_005702
+2007_005790
+2007_005797
+2007_005859
+2007_005878
+2007_005902
+2007_005951
+2007_005988
+2007_005989
+2007_006004
+2007_006066
+2007_006134
+2007_006136
+2007_006151
+2007_006212
+2007_006232
+2007_006254
+2007_006281
+2007_006303
+2007_006317
+2007_006400
+2007_006409
+2007_006445
+2007_006477
+2007_006483
+2007_006490
+2007_006530
+2007_006581
+2007_006585
+2007_006605
+2007_006615
+2007_006641
+2007_006660
+2007_006661
+2007_006673
+2007_006699
+2007_006704
+2007_006803
+2007_006832
+2007_006865
+2007_006899
+2007_006900
+2007_006944
+2007_007003
+2007_007021
+2007_007048
+2007_007098
+2007_007154
+2007_007230
+2007_007250
+2007_007355
+2007_007387
+2007_007398
+2007_007415
+2007_007432
+2007_007447
+2007_007480
+2007_007481
+2007_007523
+2007_007530
+2007_007585
+2007_007591
+2007_007621
+2007_007649
+2007_007698
+2007_007726
+2007_007772
+2007_007773
+2007_007783
+2007_007878
+2007_007890
+2007_007891
+2007_007902
+2007_007908
+2007_007930
+2007_007947
+2007_007948
+2007_008043
+2007_008072
+2007_008085
+2007_008140
+2007_008142
+2007_008203
+2007_008218
+2007_008219
+2007_008307
+2007_008403
+2007_008407
+2007_008468
+2007_008526
+2007_008571
+2007_008575
+2007_008714
+2007_008764
+2007_008778
+2007_008801
+2007_008821
+2007_008927
+2007_008932
+2007_008945
+2007_008948
+2007_008994
+2007_009030
+2007_009052
+2007_009082
+2007_009139
+2007_009209
+2007_009216
+2007_009295
+2007_009322
+2007_009327
+2007_009348
+2007_009422
+2007_009435
+2007_009436
+2007_009464
+2007_009527
+2007_009533
+2007_009550
+2007_009554
+2007_009580
+2007_009594
+2007_009597
+2007_009605
+2007_009607
+2007_009618
+2007_009630
+2007_009649
+2007_009665
+2007_009709
+2007_009724
+2007_009759
+2007_009779
+2007_009788
+2007_009807
+2007_009832
+2007_009889
+2007_009899
+2007_009901
+2007_009947
+2007_009950
+2008_000015
+2008_000019
+2008_000028
+2008_000033
+2008_000074
+2008_000089
+2008_000103
+2008_000105
+2008_000131
+2008_000144
+2008_000162
+2008_000187
+2008_000188
+2008_000197
+2008_000207
+2008_000217
+2008_000226
+2008_000235
+2008_000238
+2008_000259
+2008_000273
+2008_000284
+2008_000287
+2008_000289
+2008_000290
+2008_000309
+2008_000316
+2008_000336
+2008_000348
+2008_000361
+2008_000365
+2008_000399
+2008_000400
+2008_000415
+2008_000422
+2008_000436
+2008_000470
+2008_000491
+2008_000495
+2008_000505
+2008_000515
+2008_000540
+2008_000544
+2008_000567
+2008_000578
+2008_000584
+2008_000588
+2008_000595
+2008_000626
+2008_000645
+2008_000676
+2008_000696
+2008_000711
+2008_000716
+2008_000733
+2008_000760
+2008_000764
+2008_000778
+2008_000785
+2008_000832
+2008_000841
+2008_000860
+2008_000861
+2008_000870
+2008_000923
+2008_001030
+2008_001056
+2008_001106
+2008_001112
+2008_001118
+2008_001119
+2008_001137
+2008_001159
+2008_001169
+2008_001188
+2008_001203
+2008_001208
+2008_001215
+2008_001235
+2008_001245
+2008_001263
+2008_001274
+2008_001358
+2008_001375
+2008_001387
+2008_001399
+2008_001402
+2008_001408
+2008_001413
+2008_001462
+2008_001467
+2008_001479
+2008_001498
+2008_001510
+2008_001523
+2008_001566
+2008_001592
+2008_001601
+2008_001610
+2008_001632
+2008_001643
+2008_001691
+2008_001716
+2008_001719
+2008_001741
+2008_001761
+2008_001787
+2008_001829
+2008_001876
+2008_001882
+2008_001896
+2008_001926
+2008_001997
+2008_002032
+2008_002064
+2008_002066
+2008_002067
+2008_002073
+2008_002079
+2008_002080
+2008_002123
+2008_002160
+2008_002175
+2008_002177
+2008_002182
+2008_002200
+2008_002210
+2008_002215
+2008_002218
+2008_002221
+2008_002247
+2008_002248
+2008_002255
+2008_002258
+2008_002288
+2008_002338
+2008_002411
+2008_002425
+2008_002471
+2008_002473
+2008_002551
+2008_002641
+2008_002650
+2008_002697
+2008_002704
+2008_002710
+2008_002719
+2008_002749
+2008_002762
+2008_002772
+2008_002834
+2008_002868
+2008_002885
+2008_002894
+2008_002960
+2008_002970
+2008_002972
+2008_002993
+2008_003060
+2008_003065
+2008_003068
+2008_003083
+2008_003087
+2008_003094
+2008_003101
+2008_003168
+2008_003180
+2008_003196
+2008_003200
+2008_003208
+2008_003252
+2008_003329
+2008_003362
+2008_003373
+2008_003381
+2008_003415
+2008_003429
+2008_003480
+2008_003500
+2008_003523
+2008_003562
+2008_003585
+2008_003665
+2008_003691
+2008_003701
+2008_003703
+2008_003729
+2008_003769
+2008_003774
+2008_003779
+2008_003814
+2008_003913
+2008_003939
+2008_003947
+2008_003986
+2008_003998
+2008_004014
+2008_004026
+2008_004055
+2008_004080
+2008_004097
+2008_004112
+2008_004259
+2008_004321
+2008_004358
+2008_004365
+2008_004416
+2008_004430
+2008_004441
+2008_004547
+2008_004551
+2008_004583
+2008_004588
+2008_004607
+2008_004663
+2008_004750
+2008_004776
+2008_004822
+2008_004838
+2008_004841
+2008_004869
+2008_004892
+2008_004911
+2008_004914
+2008_004946
+2008_004983
+2008_005006
+2008_005074
+2008_005196
+2008_005214
+2008_005231
+2008_005266
+2008_005294
+2008_005300
+2008_005321
+2008_005342
+2008_005345
+2008_005367
+2008_005375
+2008_005512
+2008_005541
+2008_005600
+2008_005650
+2008_005668
+2008_005678
+2008_005679
+2008_005698
+2008_005706
+2008_005713
+2008_005714
+2008_005716
+2008_005747
+2008_005770
+2008_005839
+2008_005843
+2008_005845
+2008_005874
+2008_005926
+2008_005938
+2008_005945
+2008_005953
+2008_006032
+2008_006065
+2008_006070
+2008_006140
+2008_006182
+2008_006213
+2008_006215
+2008_006221
+2008_006289
+2008_006339
+2008_006345
+2008_006349
+2008_006353
+2008_006389
+2008_006434
+2008_006481
+2008_006482
+2008_006490
+2008_006509
+2008_006558
+2008_006655
+2008_006748
+2008_006751
+2008_006843
+2008_006873
+2008_006877
+2008_006908
+2008_006920
+2008_007011
+2008_007012
+2008_007090
+2008_007142
+2008_007165
+2008_007201
+2008_007239
+2008_007242
+2008_007245
+2008_007313
+2008_007355
+2008_007357
+2008_007375
+2008_007428
+2008_007433
+2008_007472
+2008_007581
+2008_007691
+2008_007759
+2008_007858
+2008_007998
+2008_008106
+2008_008193
+2008_008263
+2008_008323
+2008_008324
+2008_008343
+2008_008462
+2008_008476
+2008_008511
+2008_008521
+2008_008525
+2008_008541
+2008_008545
+2008_008550
+2008_008770
+2008_008773
+2009_000006
+2009_000015
+2009_000028
+2009_000029
+2009_000073
+2009_000100
+2009_000103
+2009_000133
+2009_000161
+2009_000176
+2009_000177
+2009_000250
+2009_000285
+2009_000347
+2009_000385
+2009_000400
+2009_000405
+2009_000408
+2009_000409
+2009_000420
+2009_000444
+2009_000454
+2009_000503
+2009_000505
+2009_000532
+2009_000535
+2009_000544
+2009_000553
+2009_000562
+2009_000603
+2009_000626
+2009_000635
+2009_000655
+2009_000662
+2009_000684
+2009_000690
+2009_000709
+2009_000720
+2009_000744
+2009_000746
+2009_000774
+2009_000801
+2009_000887
+2009_000894
+2009_000895
+2009_000906
+2009_000938
+2009_000987
+2009_000996
+2009_001002
+2009_001019
+2009_001027
+2009_001036
+2009_001070
+2009_001085
+2009_001095
+2009_001096
+2009_001100
+2009_001104
+2009_001117
+2009_001124
+2009_001137
+2009_001140
+2009_001145
+2009_001146
+2009_001163
+2009_001177
+2009_001197
+2009_001203
+2009_001205
+2009_001251
+2009_001253
+2009_001264
+2009_001268
+2009_001270
+2009_001283
+2009_001306
+2009_001311
+2009_001339
+2009_001359
+2009_001385
+2009_001388
+2009_001390
+2009_001403
+2009_001422
+2009_001443
+2009_001444
+2009_001481
+2009_001502
+2009_001514
+2009_001516
+2009_001544
+2009_001615
+2009_001625
+2009_001636
+2009_001640
+2009_001651
+2009_001664
+2009_001690
+2009_001693
+2009_001724
+2009_001735
+2009_001744
+2009_001755
+2009_001782
+2009_001783
+2009_001802
+2009_001828
+2009_001868
+2009_001871
+2009_001885
+2009_001888
+2009_001894
+2009_001898
+2009_001922
+2009_001937
+2009_001961
+2009_001964
+2009_001972
+2009_002010
+2009_002019
+2009_002052
+2009_002060
+2009_002072
+2009_002083
+2009_002117
+2009_002153
+2009_002204
+2009_002216
+2009_002229
+2009_002245
+2009_002262
+2009_002264
+2009_002281
+2009_002285
+2009_002314
+2009_002343
+2009_002362
+2009_002387
+2009_002409
+2009_002416
+2009_002419
+2009_002422
+2009_002423
+2009_002425
+2009_002448
+2009_002460
+2009_002472
+2009_002519
+2009_002530
+2009_002543
+2009_002567
+2009_002586
+2009_002588
+2009_002599
+2009_002613
+2009_002626
+2009_002628
+2009_002662
+2009_002674
+2009_002713
+2009_002715
+2009_002734
+2009_002763
+2009_002789
+2009_002820
+2009_002844
+2009_002845
+2009_002849
+2009_002862
+2009_002872
+2009_002885
+2009_002897
+2009_002912
+2009_002914
+2009_002917
+2009_002932
+2009_002972
+2009_002984
+2009_002988
+2009_002993
+2009_003006
+2009_003007
+2009_003012
+2009_003034
+2009_003035
+2009_003039
+2009_003053
+2009_003054
+2009_003075
+2009_003087
+2009_003088
+2009_003090
+2009_003142
+2009_003146
+2009_003147
+2009_003164
+2009_003172
+2009_003200
+2009_003249
+2009_003317
+2009_003340
+2009_003345
+2009_003353
+2009_003361
+2009_003369
+2009_003455
+2009_003461
+2009_003468
+2009_003497
+2009_003519
+2009_003522
+2009_003539
+2009_003555
+2009_003613
+2009_003636
+2009_003646
+2009_003660
+2009_003690
+2009_003697
+2009_003711
+2009_003734
+2009_003736
+2009_003757
+2009_003768
+2009_003783
+2009_003799
+2009_003815
+2009_003820
+2009_003825
+2009_003860
+2009_003865
+2009_003921
+2009_003922
+2009_003933
+2009_003961
+2009_003975
+2009_004091
+2009_004095
+2009_004105
+2009_004117
+2009_004171
+2009_004178
+2009_004180
+2009_004186
+2009_004191
+2009_004212
+2009_004213
+2009_004228
+2009_004249
+2009_004264
+2009_004278
+2009_004301
+2009_004316
+2009_004317
+2009_004327
+2009_004328
+2009_004334
+2009_004336
+2009_004368
+2009_004374
+2009_004409
+2009_004417
+2009_004425
+2009_004426
+2009_004434
+2009_004446
+2009_004464
+2009_004479
+2009_004519
+2009_004539
+2009_004561
+2009_004620
+2009_004626
+2009_004643
+2009_004656
+2009_004661
+2009_004674
+2009_004705
+2009_004790
+2009_004805
+2009_004829
+2009_004887
+2009_004888
+2009_004890
+2009_004901
+2009_004904
+2009_004919
+2009_004939
+2009_004980
+2009_004990
+2009_005000
+2009_005016
+2009_005031
+2009_005037
+2009_005055
+2009_005056
+2009_005069
+2009_005084
+2009_005085
+2009_005107
+2009_005118
+2009_005120
+2009_005128
+2009_005130
+2009_005141
+2009_005145
+2009_005160
+2009_005177
+2009_005194
+2009_005234
+2009_005236
+2009_005247
+2009_005269
+2009_005287
+2010_000002
+2010_000043
+2010_000063
+2010_000075
+2010_000076
+2010_000114
+2010_000117
+2010_000131
+2010_000132
+2010_000148
+2010_000187
+2010_000189
+2010_000195
+2010_000269
+2010_000285
+2010_000371
+2010_000392
+2010_000404
+2010_000436
+2010_000437
+2010_000466
+2010_000469
+2010_000492
+2010_000498
+2010_000503
+2010_000519
+2010_000567
+2010_000588
+2010_000632
+2010_000661
+2010_000675
+2010_000685
+2010_000746
+2010_000748
+2010_000772
+2010_000787
+2010_000810
+2010_000815
+2010_000847
+2010_000855
+2010_000885
+2010_000887
+2010_000978
+2010_000986
+2010_001043
+2010_001120
+2010_001131
+2010_001154
+2010_001160
+2010_001177
+2010_001183
+2010_001184
+2010_001195
+2010_001245
+2010_001247
+2010_001261
+2010_001273
+2010_001279
+2010_001282
+2010_001329
+2010_001347
+2010_001374
+2010_001386
+2010_001399
+2010_001413
+2010_001418
+2010_001422
+2010_001457
+2010_001514
+2010_001515
+2010_001561
+2010_001562
+2010_001576
+2010_001590
+2010_001595
+2010_001618
+2010_001619
+2010_001630
+2010_001660
+2010_001676
+2010_001706
+2010_001732
+2010_001748
+2010_001807
+2010_001842
+2010_001849
+2010_001850
+2010_001852
+2010_001860
+2010_001922
+2010_001923
+2010_001933
+2010_001939
+2010_001944
+2010_002018
+2010_002020
+2010_002032
+2010_002039
+2010_002047
+2010_002054
+2010_002055
+2010_002070
+2010_002097
+2010_002107
+2010_002139
+2010_002154
+2010_002166
+2010_002203
+2010_002218
+2010_002236
+2010_002254
+2010_002286
+2010_002338
+2010_002363
+2010_002379
+2010_002382
+2010_002387
+2010_002413
+2010_002418
+2010_002440
+2010_002455
+2010_002457
+2010_002499
+2010_002527
+2010_002532
+2010_002551
+2010_002556
+2010_002570
+2010_002573
+2010_002625
+2010_002659
+2010_002697
+2010_002720
+2010_002733
+2010_002750
+2010_002778
+2010_002786
+2010_002794
+2010_002811
+2010_002815
+2010_002838
+2010_002856
+2010_002870
+2010_002892
+2010_002907
+2010_002935
+2010_002937
+2010_002938
+2010_002962
+2010_002973
+2010_003010
+2010_003017
+2010_003062
+2010_003088
+2010_003093
+2010_003097
+2010_003114
+2010_003119
+2010_003153
+2010_003157
+2010_003170
+2010_003174
+2010_003203
+2010_003230
+2010_003250
+2010_003252
+2010_003269
+2010_003274
+2010_003342
+2010_003345
+2010_003380
+2010_003383
+2010_003384
+2010_003529
+2010_003534
+2010_003599
+2010_003634
+2010_003651
+2010_003665
+2010_003670
+2010_003680
+2010_003696
+2010_003717
+2010_003737
+2010_003798
+2010_003799
+2010_003884
+2010_003887
+2010_003894
+2010_003899
+2010_003911
+2010_003925
+2010_003950
+2010_003954
+2010_003958
+2010_003974
+2010_004005
+2010_004025
+2010_004060
+2010_004069
+2010_004071
+2010_004072
+2010_004074
+2010_004109
+2010_004119
+2010_004144
+2010_004154
+2010_004171
+2010_004180
+2010_004186
+2010_004210
+2010_004222
+2010_004258
+2010_004283
+2010_004288
+2010_004289
+2010_004306
+2010_004361
+2010_004363
+2010_004365
+2010_004370
+2010_004429
+2010_004450
+2010_004478
+2010_004481
+2010_004493
+2010_004499
+2010_004540
+2010_004560
+2010_004577
+2010_004598
+2010_004616
+2010_004620
+2010_004625
+2010_004669
+2010_004683
+2010_004694
+2010_004704
+2010_004721
+2010_004760
+2010_004766
+2010_004773
+2010_004805
+2010_004808
+2010_004900
+2010_004916
+2010_004933
+2010_004938
+2010_004948
+2010_004960
+2010_004963
+2010_005016
+2010_005028
+2010_005055
+2010_005064
+2010_005098
+2010_005106
+2010_005111
+2010_005119
+2010_005128
+2010_005129
+2010_005198
+2010_005202
+2010_005217
+2010_005223
+2010_005232
+2010_005277
+2010_005317
+2010_005318
+2010_005419
+2010_005429
+2010_005450
+2010_005457
+2010_005468
+2010_005471
+2010_005494
+2010_005500
+2010_005505
+2010_005506
+2010_005513
+2010_005519
+2010_005522
+2010_005596
+2010_005627
+2010_005643
+2010_005652
+2010_005663
+2010_005669
+2010_005678
+2010_005700
+2010_005721
+2010_005723
+2010_005725
+2010_005734
+2010_005744
+2010_005746
+2010_005755
+2010_005758
+2010_005775
+2010_005791
+2010_005796
+2010_005800
+2010_005805
+2010_005810
+2010_005820
+2010_005830
+2010_005835
+2010_005836
+2010_005876
+2010_005891
+2010_005898
+2010_005919
+2010_005927
+2010_005932
+2010_005951
+2010_005952
+2010_005978
+2010_005982
+2010_006009
+2011_000003
+2011_000006
+2011_000025
+2011_000027
+2011_000068
+2011_000069
+2011_000105
+2011_000108
+2011_000116
+2011_000122
+2011_000145
+2011_000149
+2011_000152
+2011_000182
+2011_000197
+2011_000208
+2011_000216
+2011_000219
+2011_000221
+2011_000222
+2011_000228
+2011_000243
+2011_000252
+2011_000258
+2011_000268
+2011_000277
+2011_000278
+2011_000293
+2011_000345
+2011_000359
+2011_000379
+2011_000382
+2011_000400
+2011_000428
+2011_000449
+2011_000453
+2011_000457
+2011_000468
+2011_000469
+2011_000513
+2011_000542
+2011_000550
+2011_000551
+2011_000556
+2011_000573
+2011_000577
+2011_000589
+2011_000594
+2011_000637
+2011_000641
+2011_000642
+2011_000646
+2011_000651
+2011_000652
+2011_000713
+2011_000758
+2011_000768
+2011_000771
+2011_000790
+2011_000793
+2011_000834
+2011_000840
+2011_000882
+2011_000893
+2011_000895
+2011_000920
+2011_000934
+2011_000944
+2011_000973
+2011_000982
+2011_000997
+2011_000999
+2011_001004
+2011_001015
+2011_001027
+2011_001133
+2011_001135
+2011_001139
+2011_001166
+2011_001175
+2011_001198
+2011_001211
+2011_001259
+2011_001270
+2011_001336
+2011_001400
+2011_001402
+2011_001411
+2011_001412
+2011_001432
+2011_001463
+2011_001475
+2011_001479
+2011_001519
+2011_001536
+2011_001542
+2011_001571
+2011_001621
+2011_001622
+2011_001632
+2011_001652
+2011_001653
+2011_001695
+2011_001710
+2011_001730
+2011_001753
+2011_001754
+2011_001764
+2011_001765
+2011_001790
+2011_001810
+2011_001855
+2011_001866
+2011_001875
+2011_001895
+2011_001902
+2011_001904
+2011_001922
+2011_001924
+2011_001928
+2011_001959
+2011_001967
+2011_001972
+2011_001974
+2011_001991
+2011_002027
+2011_002050
+2011_002107
+2011_002111
+2011_002114
+2011_002119
+2011_002134
+2011_002135
+2011_002149
+2011_002222
+2011_002224
+2011_002227
+2011_002246
+2011_002291
+2011_002300
+2011_002303
+2011_002335
+2011_002341
+2011_002350
+2011_002381
+2011_002385
+2011_002389
+2011_002398
+2011_002410
+2011_002447
+2011_002457
+2011_002464
+2011_002488
+2011_002503
+2011_002504
+2011_002511
+2011_002528
+2011_002553
+2011_002559
+2011_002561
+2011_002585
+2011_002590
+2011_002652
+2011_002656
+2011_002709
+2011_002715
+2011_002717
+2011_002752
+2011_002767
+2011_002770
+2011_002834
+2011_002851
+2011_002872
+2011_002873
+2011_002920
+2011_002932
+2011_002935
+2011_002947
+2011_002953
+2011_002956
+2011_003025
+2011_003038
+2011_003057
+2011_003066
+2011_003078
+2011_003121
+2011_003141
+2011_003151
+2011_003184
+2011_003216
+2011_003238
+2011_003246
+2011_003255

data/val.txt

@@ -0,0 +1,1449 @@
+2007_000033
+2007_000042
+2007_000061
+2007_000123
+2007_000129
+2007_000175
+2007_000187
+2007_000323
+2007_000332
+2007_000346
+2007_000452
+2007_000464
+2007_000491
+2007_000529
+2007_000559
+2007_000572
+2007_000629
+2007_000636
+2007_000661
+2007_000663
+2007_000676
+2007_000727
+2007_000762
+2007_000783
+2007_000799
+2007_000804
+2007_000830
+2007_000837
+2007_000847
+2007_000862
+2007_000925
+2007_000999
+2007_001154
+2007_001175
+2007_001239
+2007_001284
+2007_001288
+2007_001289
+2007_001299
+2007_001311
+2007_001321
+2007_001377
+2007_001408
+2007_001423
+2007_001430
+2007_001457
+2007_001458
+2007_001526
+2007_001568
+2007_001585
+2007_001586
+2007_001587
+2007_001594
+2007_001630
+2007_001677
+2007_001678
+2007_001717
+2007_001733
+2007_001761
+2007_001763
+2007_001774
+2007_001884
+2007_001955
+2007_002046
+2007_002094
+2007_002119
+2007_002132
+2007_002260
+2007_002266
+2007_002268
+2007_002284
+2007_002376
+2007_002378
+2007_002387
+2007_002400
+2007_002412
+2007_002426
+2007_002427
+2007_002445
+2007_002470
+2007_002539
+2007_002565
+2007_002597
+2007_002618
+2007_002619
+2007_002624
+2007_002643
+2007_002648
+2007_002719
+2007_002728
+2007_002823
+2007_002824
+2007_002852
+2007_002903
+2007_003011
+2007_003020
+2007_003022
+2007_003051
+2007_003088
+2007_003101
+2007_003106
+2007_003110
+2007_003131
+2007_003134
+2007_003137
+2007_003143
+2007_003169
+2007_003188
+2007_003194
+2007_003195
+2007_003201
+2007_003349
+2007_003367
+2007_003373
+2007_003499
+2007_003503
+2007_003506
+2007_003530
+2007_003571
+2007_003587
+2007_003611
+2007_003621
+2007_003682
+2007_003711
+2007_003714
+2007_003742
+2007_003786
+2007_003841
+2007_003848
+2007_003861
+2007_003872
+2007_003917
+2007_003957
+2007_003991
+2007_004033
+2007_004052
+2007_004112
+2007_004121
+2007_004143
+2007_004189
+2007_004190
+2007_004193
+2007_004241
+2007_004275
+2007_004281
+2007_004380
+2007_004392
+2007_004405
+2007_004468
+2007_004483
+2007_004510
+2007_004538
+2007_004558
+2007_004644
+2007_004649
+2007_004712
+2007_004722
+2007_004856
+2007_004866
+2007_004902
+2007_004969
+2007_005058
+2007_005074
+2007_005107
+2007_005114
+2007_005149
+2007_005173
+2007_005281
+2007_005294
+2007_005296
+2007_005304
+2007_005331
+2007_005354
+2007_005358
+2007_005428
+2007_005460
+2007_005469
+2007_005509
+2007_005547
+2007_005600
+2007_005608
+2007_005626
+2007_005689
+2007_005696
+2007_005705
+2007_005759
+2007_005803
+2007_005813
+2007_005828
+2007_005844
+2007_005845
+2007_005857
+2007_005911
+2007_005915
+2007_005978
+2007_006028
+2007_006035
+2007_006046
+2007_006076
+2007_006086
+2007_006117
+2007_006171
+2007_006241
+2007_006260
+2007_006277
+2007_006348
+2007_006364
+2007_006373
+2007_006444
+2007_006449
+2007_006549
+2007_006553
+2007_006560
+2007_006647
+2007_006678
+2007_006680
+2007_006698
+2007_006761
+2007_006802
+2007_006837
+2007_006841
+2007_006864
+2007_006866
+2007_006946
+2007_007007
+2007_007084
+2007_007109
+2007_007130
+2007_007165
+2007_007168
+2007_007195
+2007_007196
+2007_007203
+2007_007211
+2007_007235
+2007_007341
+2007_007414
+2007_007417
+2007_007470
+2007_007477
+2007_007493
+2007_007498
+2007_007524
+2007_007534
+2007_007624
+2007_007651
+2007_007688
+2007_007748
+2007_007795
+2007_007810
+2007_007815
+2007_007818
+2007_007836
+2007_007849
+2007_007881
+2007_007996
+2007_008051
+2007_008084
+2007_008106
+2007_008110
+2007_008204
+2007_008222
+2007_008256
+2007_008260
+2007_008339
+2007_008374
+2007_008415
+2007_008430
+2007_008543
+2007_008547
+2007_008596
+2007_008645
+2007_008670
+2007_008708
+2007_008722
+2007_008747
+2007_008802
+2007_008815
+2007_008897
+2007_008944
+2007_008964
+2007_008973
+2007_008980
+2007_009015
+2007_009068
+2007_009084
+2007_009088
+2007_009096
+2007_009221
+2007_009245
+2007_009251
+2007_009252
+2007_009258
+2007_009320
+2007_009323
+2007_009331
+2007_009346
+2007_009392
+2007_009413
+2007_009419
+2007_009446
+2007_009458
+2007_009521
+2007_009562
+2007_009592
+2007_009654
+2007_009655
+2007_009684
+2007_009687
+2007_009691
+2007_009706
+2007_009750
+2007_009756
+2007_009764
+2007_009794
+2007_009817
+2007_009841
+2007_009897
+2007_009911
+2007_009923
+2007_009938
+2008_000009
+2008_000016
+2008_000073
+2008_000075
+2008_000080
+2008_000107
+2008_000120
+2008_000123
+2008_000149
+2008_000182
+2008_000213
+2008_000215
+2008_000223
+2008_000233
+2008_000234
+2008_000239
+2008_000254
+2008_000270
+2008_000271
+2008_000345
+2008_000359
+2008_000391
+2008_000401
+2008_000464
+2008_000469
+2008_000474
+2008_000501
+2008_000510
+2008_000533
+2008_000573
+2008_000589
+2008_000602
+2008_000630
+2008_000657
+2008_000661
+2008_000662
+2008_000666
+2008_000673
+2008_000700
+2008_000725
+2008_000731
+2008_000763
+2008_000765
+2008_000782
+2008_000795
+2008_000811
+2008_000848
+2008_000853
+2008_000863
+2008_000911
+2008_000919
+2008_000943
+2008_000992
+2008_001013
+2008_001028
+2008_001040
+2008_001070
+2008_001074
+2008_001076
+2008_001078
+2008_001135
+2008_001150
+2008_001170
+2008_001231
+2008_001249
+2008_001260
+2008_001283
+2008_001308
+2008_001379
+2008_001404
+2008_001433
+2008_001439
+2008_001478
+2008_001491
+2008_001504
+2008_001513
+2008_001514
+2008_001531
+2008_001546
+2008_001547
+2008_001580
+2008_001629
+2008_001640
+2008_001682
+2008_001688
+2008_001715
+2008_001821
+2008_001874
+2008_001885
+2008_001895
+2008_001966
+2008_001971
+2008_001992
+2008_002043
+2008_002152
+2008_002205
+2008_002212
+2008_002239
+2008_002240
+2008_002241
+2008_002269
+2008_002273
+2008_002358
+2008_002379
+2008_002383
+2008_002429
+2008_002464
+2008_002467
+2008_002492
+2008_002495
+2008_002504
+2008_002521
+2008_002536
+2008_002588
+2008_002623
+2008_002680
+2008_002681
+2008_002775
+2008_002778
+2008_002835
+2008_002859
+2008_002864
+2008_002900
+2008_002904
+2008_002929
+2008_002936
+2008_002942
+2008_002958
+2008_003003
+2008_003026
+2008_003034
+2008_003076
+2008_003105
+2008_003108
+2008_003110
+2008_003135
+2008_003141
+2008_003155
+2008_003210
+2008_003238
+2008_003270
+2008_003330
+2008_003333
+2008_003369
+2008_003379
+2008_003451
+2008_003461
+2008_003477
+2008_003492
+2008_003499
+2008_003511
+2008_003546
+2008_003576
+2008_003577
+2008_003676
+2008_003709
+2008_003733
+2008_003777
+2008_003782
+2008_003821
+2008_003846
+2008_003856
+2008_003858
+2008_003874
+2008_003876
+2008_003885
+2008_003886
+2008_003926
+2008_003976
+2008_004069
+2008_004101
+2008_004140
+2008_004172
+2008_004175
+2008_004212
+2008_004279
+2008_004339
+2008_004345
+2008_004363
+2008_004367
+2008_004396
+2008_004399
+2008_004453
+2008_004477
+2008_004552
+2008_004562
+2008_004575
+2008_004610
+2008_004612
+2008_004621
+2008_004624
+2008_004654
+2008_004659
+2008_004687
+2008_004701
+2008_004704
+2008_004705
+2008_004754
+2008_004758
+2008_004854
+2008_004910
+2008_004995
+2008_005049
+2008_005089
+2008_005097
+2008_005105
+2008_005145
+2008_005197
+2008_005217
+2008_005242
+2008_005245
+2008_005254
+2008_005262
+2008_005338
+2008_005398
+2008_005399
+2008_005422
+2008_005439
+2008_005445
+2008_005525
+2008_005544
+2008_005628
+2008_005633
+2008_005637
+2008_005642
+2008_005676
+2008_005680
+2008_005691
+2008_005727
+2008_005738
+2008_005812
+2008_005904
+2008_005915
+2008_006008
+2008_006036
+2008_006055
+2008_006063
+2008_006108
+2008_006130
+2008_006143
+2008_006159
+2008_006216
+2008_006219
+2008_006229
+2008_006254
+2008_006275
+2008_006325
+2008_006327
+2008_006341
+2008_006408
+2008_006480
+2008_006523
+2008_006526
+2008_006528
+2008_006553
+2008_006554
+2008_006703
+2008_006722
+2008_006752
+2008_006784
+2008_006835
+2008_006874
+2008_006981
+2008_006986
+2008_007025
+2008_007031
+2008_007048
+2008_007120
+2008_007123
+2008_007143
+2008_007194
+2008_007219
+2008_007273
+2008_007350
+2008_007378
+2008_007392
+2008_007402
+2008_007497
+2008_007498
+2008_007507
+2008_007513
+2008_007527
+2008_007548
+2008_007596
+2008_007677
+2008_007737
+2008_007797
+2008_007804
+2008_007811
+2008_007814
+2008_007828
+2008_007836
+2008_007945
+2008_007994
+2008_008051
+2008_008103
+2008_008127
+2008_008221
+2008_008252
+2008_008268
+2008_008296
+2008_008301
+2008_008335
+2008_008362
+2008_008392
+2008_008393
+2008_008421
+2008_008434
+2008_008469
+2008_008629
+2008_008682
+2008_008711
+2008_008746
+2009_000012
+2009_000013
+2009_000022
+2009_000032
+2009_000037
+2009_000039
+2009_000074
+2009_000080
+2009_000087
+2009_000096
+2009_000121
+2009_000136
+2009_000149
+2009_000156
+2009_000201
+2009_000205
+2009_000219
+2009_000242
+2009_000309
+2009_000318
+2009_000335
+2009_000351
+2009_000354
+2009_000387
+2009_000391
+2009_000412
+2009_000418
+2009_000421
+2009_000426
+2009_000440
+2009_000446
+2009_000455
+2009_000457
+2009_000469
+2009_000487
+2009_000488
+2009_000523
+2009_000573
+2009_000619
+2009_000628
+2009_000641
+2009_000664
+2009_000675
+2009_000704
+2009_000705
+2009_000712
+2009_000716
+2009_000723
+2009_000727
+2009_000730
+2009_000731
+2009_000732
+2009_000771
+2009_000825
+2009_000828
+2009_000839
+2009_000840
+2009_000845
+2009_000879
+2009_000892
+2009_000919
+2009_000924
+2009_000931
+2009_000935
+2009_000964
+2009_000989
+2009_000991
+2009_000998
+2009_001008
+2009_001082
+2009_001108
+2009_001160
+2009_001215
+2009_001240
+2009_001255
+2009_001278
+2009_001299
+2009_001300
+2009_001314
+2009_001332
+2009_001333
+2009_001363
+2009_001391
+2009_001411
+2009_001433
+2009_001505
+2009_001535
+2009_001536
+2009_001565
+2009_001607
+2009_001644
+2009_001663
+2009_001683
+2009_001684
+2009_001687
+2009_001718
+2009_001731
+2009_001765
+2009_001768
+2009_001775
+2009_001804
+2009_001816
+2009_001818
+2009_001850
+2009_001851
+2009_001854
+2009_001941
+2009_001991
+2009_002012
+2009_002035
+2009_002042
+2009_002082
+2009_002094
+2009_002097
+2009_002122
+2009_002150
+2009_002155
+2009_002164
+2009_002165
+2009_002171
+2009_002185
+2009_002202
+2009_002221
+2009_002238
+2009_002239
+2009_002265
+2009_002268
+2009_002291
+2009_002295
+2009_002317
+2009_002320
+2009_002346
+2009_002366
+2009_002372
+2009_002382
+2009_002390
+2009_002415
+2009_002445
+2009_002487
+2009_002521
+2009_002527
+2009_002535
+2009_002539
+2009_002549
+2009_002562
+2009_002568
+2009_002571
+2009_002573
+2009_002584
+2009_002591
+2009_002594
+2009_002604
+2009_002618
+2009_002635
+2009_002638
+2009_002649
+2009_002651
+2009_002727
+2009_002732
+2009_002749
+2009_002753
+2009_002771
+2009_002808
+2009_002856
+2009_002887
+2009_002888
+2009_002928
+2009_002936
+2009_002975
+2009_002982
+2009_002990
+2009_003003
+2009_003005
+2009_003043
+2009_003059
+2009_003063
+2009_003065
+2009_003071
+2009_003080
+2009_003105
+2009_003123
+2009_003193
+2009_003196
+2009_003217
+2009_003224
+2009_003241
+2009_003269
+2009_003273
+2009_003299
+2009_003304
+2009_003311
+2009_003323
+2009_003343
+2009_003378
+2009_003387
+2009_003406
+2009_003433
+2009_003450
+2009_003466
+2009_003481
+2009_003494
+2009_003498
+2009_003504
+2009_003507
+2009_003517
+2009_003523
+2009_003542
+2009_003549
+2009_003551
+2009_003564
+2009_003569
+2009_003576
+2009_003589
+2009_003607
+2009_003640
+2009_003666
+2009_003696
+2009_003703
+2009_003707
+2009_003756
+2009_003771
+2009_003773
+2009_003804
+2009_003806
+2009_003810
+2009_003849
+2009_003857
+2009_003858
+2009_003895
+2009_003903
+2009_003904
+2009_003928
+2009_003938
+2009_003971
+2009_003991
+2009_004021
+2009_004033
+2009_004043
+2009_004070
+2009_004072
+2009_004084
+2009_004099
+2009_004125
+2009_004140
+2009_004217
+2009_004221
+2009_004247
+2009_004248
+2009_004255
+2009_004298
+2009_004324
+2009_004455
+2009_004494
+2009_004497
+2009_004504
+2009_004507
+2009_004509
+2009_004540
+2009_004568
+2009_004579
+2009_004581
+2009_004590
+2009_004592
+2009_004594
+2009_004635
+2009_004653
+2009_004687
+2009_004721
+2009_004730
+2009_004732
+2009_004738
+2009_004748
+2009_004789
+2009_004799
+2009_004801
+2009_004848
+2009_004859
+2009_004867
+2009_004882
+2009_004886
+2009_004895
+2009_004942
+2009_004969
+2009_004987
+2009_004993
+2009_004994
+2009_005038
+2009_005078
+2009_005087
+2009_005089
+2009_005137
+2009_005148
+2009_005156
+2009_005158
+2009_005189
+2009_005190
+2009_005217
+2009_005219
+2009_005220
+2009_005231
+2009_005260
+2009_005262
+2009_005302
+2010_000003
+2010_000038
+2010_000065
+2010_000083
+2010_000084
+2010_000087
+2010_000110
+2010_000159
+2010_000160
+2010_000163
+2010_000174
+2010_000216
+2010_000238
+2010_000241
+2010_000256
+2010_000272
+2010_000284
+2010_000309
+2010_000318
+2010_000330
+2010_000335
+2010_000342
+2010_000372
+2010_000422
+2010_000426
+2010_000427
+2010_000502
+2010_000530
+2010_000552
+2010_000559
+2010_000572
+2010_000573
+2010_000622
+2010_000628
+2010_000639
+2010_000666
+2010_000679
+2010_000682
+2010_000683
+2010_000724
+2010_000738
+2010_000764
+2010_000788
+2010_000814
+2010_000836
+2010_000874
+2010_000904
+2010_000906
+2010_000907
+2010_000918
+2010_000929
+2010_000941
+2010_000952
+2010_000961
+2010_001000
+2010_001010
+2010_001011
+2010_001016
+2010_001017
+2010_001024
+2010_001036
+2010_001061
+2010_001069
+2010_001070
+2010_001079
+2010_001104
+2010_001124
+2010_001149
+2010_001151
+2010_001174
+2010_001206
+2010_001246
+2010_001251
+2010_001256
+2010_001264
+2010_001292
+2010_001313
+2010_001327
+2010_001331
+2010_001351
+2010_001367
+2010_001376
+2010_001403
+2010_001448
+2010_001451
+2010_001522
+2010_001534
+2010_001553
+2010_001557
+2010_001563
+2010_001577
+2010_001579
+2010_001646
+2010_001656
+2010_001692
+2010_001699
+2010_001734
+2010_001752
+2010_001767
+2010_001768
+2010_001773
+2010_001820
+2010_001830
+2010_001851
+2010_001908
+2010_001913
+2010_001951
+2010_001956
+2010_001962
+2010_001966
+2010_001995
+2010_002017
+2010_002025
+2010_002030
+2010_002106
+2010_002137
+2010_002142
+2010_002146
+2010_002147
+2010_002150
+2010_002161
+2010_002200
+2010_002228
+2010_002232
+2010_002251
+2010_002271
+2010_002305
+2010_002310
+2010_002336
+2010_002348
+2010_002361
+2010_002390
+2010_002396
+2010_002422
+2010_002450
+2010_002480
+2010_002512
+2010_002531
+2010_002536
+2010_002538
+2010_002546
+2010_002623
+2010_002682
+2010_002691
+2010_002693
+2010_002701
+2010_002763
+2010_002792
+2010_002868
+2010_002900
+2010_002902
+2010_002921
+2010_002929
+2010_002939
+2010_002988
+2010_003014
+2010_003060
+2010_003123
+2010_003127
+2010_003132
+2010_003168
+2010_003183
+2010_003187
+2010_003207
+2010_003231
+2010_003239
+2010_003275
+2010_003276
+2010_003293
+2010_003302
+2010_003325
+2010_003362
+2010_003365
+2010_003381
+2010_003402
+2010_003409
+2010_003418
+2010_003446
+2010_003453
+2010_003468
+2010_003473
+2010_003495
+2010_003506
+2010_003514
+2010_003531
+2010_003532
+2010_003541
+2010_003547
+2010_003597
+2010_003675
+2010_003708
+2010_003716
+2010_003746
+2010_003758
+2010_003764
+2010_003768
+2010_003771
+2010_003772
+2010_003781
+2010_003813
+2010_003820
+2010_003854
+2010_003912
+2010_003915
+2010_003947
+2010_003956
+2010_003971
+2010_004041
+2010_004042
+2010_004056
+2010_004063
+2010_004104
+2010_004120
+2010_004149
+2010_004165
+2010_004208
+2010_004219
+2010_004226
+2010_004314
+2010_004320
+2010_004322
+2010_004337
+2010_004348
+2010_004355
+2010_004369
+2010_004382
+2010_004419
+2010_004432
+2010_004472
+2010_004479
+2010_004519
+2010_004520
+2010_004529
+2010_004543
+2010_004550
+2010_004551
+2010_004556
+2010_004559
+2010_004628
+2010_004635
+2010_004662
+2010_004697
+2010_004757
+2010_004763
+2010_004772
+2010_004783
+2010_004789
+2010_004795
+2010_004815
+2010_004825
+2010_004828
+2010_004856
+2010_004857
+2010_004861
+2010_004941
+2010_004946
+2010_004951
+2010_004980
+2010_004994
+2010_005013
+2010_005021
+2010_005046
+2010_005063
+2010_005108
+2010_005118
+2010_005159
+2010_005160
+2010_005166
+2010_005174
+2010_005180
+2010_005187
+2010_005206
+2010_005245
+2010_005252
+2010_005284
+2010_005305
+2010_005344
+2010_005353
+2010_005366
+2010_005401
+2010_005421
+2010_005428
+2010_005432
+2010_005433
+2010_005496
+2010_005501
+2010_005508
+2010_005531
+2010_005534
+2010_005575
+2010_005582
+2010_005606
+2010_005626
+2010_005644
+2010_005664
+2010_005705
+2010_005706
+2010_005709
+2010_005718
+2010_005719
+2010_005727
+2010_005762
+2010_005788
+2010_005860
+2010_005871
+2010_005877
+2010_005888
+2010_005899
+2010_005922
+2010_005991
+2010_005992
+2010_006026
+2010_006034
+2010_006054
+2010_006070
+2011_000045
+2011_000051
+2011_000054
+2011_000066
+2011_000070
+2011_000112
+2011_000173
+2011_000178
+2011_000185
+2011_000226
+2011_000234
+2011_000238
+2011_000239
+2011_000248
+2011_000283
+2011_000291
+2011_000310
+2011_000312
+2011_000338
+2011_000396
+2011_000412
+2011_000419
+2011_000435
+2011_000436
+2011_000438
+2011_000455
+2011_000456
+2011_000479
+2011_000481
+2011_000482
+2011_000503
+2011_000512
+2011_000521
+2011_000526
+2011_000536
+2011_000548
+2011_000566
+2011_000585
+2011_000598
+2011_000607
+2011_000618
+2011_000638
+2011_000658
+2011_000661
+2011_000669
+2011_000747
+2011_000780
+2011_000789
+2011_000807
+2011_000809
+2011_000813
+2011_000830
+2011_000843
+2011_000874
+2011_000888
+2011_000900
+2011_000912
+2011_000953
+2011_000969
+2011_001005
+2011_001014
+2011_001020
+2011_001047
+2011_001060
+2011_001064
+2011_001069
+2011_001071
+2011_001082
+2011_001110
+2011_001114
+2011_001159
+2011_001161
+2011_001190
+2011_001232
+2011_001263
+2011_001276
+2011_001281
+2011_001287
+2011_001292
+2011_001313
+2011_001341
+2011_001346
+2011_001350
+2011_001407
+2011_001416
+2011_001421
+2011_001434
+2011_001447
+2011_001489
+2011_001529
+2011_001530
+2011_001534
+2011_001546
+2011_001567
+2011_001589
+2011_001597
+2011_001601
+2011_001607
+2011_001613
+2011_001614
+2011_001619
+2011_001624
+2011_001642
+2011_001665
+2011_001669
+2011_001674
+2011_001708
+2011_001713
+2011_001714
+2011_001722
+2011_001726
+2011_001745
+2011_001748
+2011_001775
+2011_001782
+2011_001793
+2011_001794
+2011_001812
+2011_001862
+2011_001863
+2011_001868
+2011_001880
+2011_001910
+2011_001984
+2011_001988
+2011_002002
+2011_002040
+2011_002041
+2011_002064
+2011_002075
+2011_002098
+2011_002110
+2011_002121
+2011_002124
+2011_002150
+2011_002156
+2011_002178
+2011_002200
+2011_002223
+2011_002244
+2011_002247
+2011_002279
+2011_002295
+2011_002298
+2011_002308
+2011_002317
+2011_002322
+2011_002327
+2011_002343
+2011_002358
+2011_002371
+2011_002379
+2011_002391
+2011_002498
+2011_002509
+2011_002515
+2011_002532
+2011_002535
+2011_002548
+2011_002575
+2011_002578
+2011_002589
+2011_002592
+2011_002623
+2011_002641
+2011_002644
+2011_002662
+2011_002675
+2011_002685
+2011_002713
+2011_002730
+2011_002754
+2011_002812
+2011_002863
+2011_002879
+2011_002885
+2011_002929
+2011_002951
+2011_002975
+2011_002993
+2011_002997
+2011_003003
+2011_003011
+2011_003019
+2011_003030
+2011_003055
+2011_003085
+2011_003103
+2011_003114
+2011_003145
+2011_003146
+2011_003182
+2011_003197
+2011_003205
+2011_003240
+2011_003256
+2011_003271