Upload 127 files

fastai/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "2.7.13"
+

fastai/_nbdev.py

@@ -0,0 +1,899 @@
+# AUTOGENERATED BY NBDEV! DO NOT EDIT!
+
+__all__ = ["index", "modules", "custom_doc_links", "git_url"]
+
+index = {"defaults.benchmark": "00_torch_core.ipynb",
+ "setup_cuda": "00_torch_core.ipynb",
+ "subplots": "00_torch_core.ipynb",
+ "show_image": "00_torch_core.ipynb",
+ "show_titled_image": "00_torch_core.ipynb",
+ "show_images": "00_torch_core.ipynb",
+ "ArrayBase": "00_torch_core.ipynb",
+ "ArrayImageBase": "00_torch_core.ipynb",
+ "ArrayImage": "00_torch_core.ipynb",
+ "ArrayImageBW": "00_torch_core.ipynb",
+ "ArrayMask": "00_torch_core.ipynb",
+ "Tensor.__array_eq__": "00_torch_core.ipynb",
+ "tensor": "00_torch_core.ipynb",
+ "set_seed": "00_torch_core.ipynb",
+ "get_random_states": "00_torch_core.ipynb",
+ "set_random_states": "00_torch_core.ipynb",
+ "no_random": "00_torch_core.ipynb",
+ "unsqueeze": "00_torch_core.ipynb",
+ "unsqueeze_": "00_torch_core.ipynb",
+ "apply": "00_torch_core.ipynb",
+ "maybe_gather": "00_torch_core.ipynb",
+ "to_detach": "00_torch_core.ipynb",
+ "to_half": "00_torch_core.ipynb",
+ "to_float": "00_torch_core.ipynb",
+ "defaults.use_cuda": "00_torch_core.ipynb",
+ "default_device": "00_torch_core.ipynb",
+ "to_device": "00_torch_core.ipynb",
+ "to_cpu": "00_torch_core.ipynb",
+ "to_np": "00_torch_core.ipynb",
+ "to_concat": "00_torch_core.ipynb",
+ "Tensor.set_meta": "00_torch_core.ipynb",
+ "Tensor.as_subclass": "00_torch_core.ipynb",
+ "TensorBase": "00_torch_core.ipynb",
+ "TensorImageBase": "00_torch_core.ipynb",
+ "TensorImage": "00_torch_core.ipynb",
+ "TensorImageBW": "00_torch_core.ipynb",
+ "TensorMask": "00_torch_core.ipynb",
+ "TensorFlowField": "00_torch_core.ipynb",
+ "TensorCategory": "00_torch_core.ipynb",
+ "TensorMultiCategory": "00_torch_core.ipynb",
+ "TitledTensorScalar": "00_torch_core.ipynb",
+ "L.tensored": "00_torch_core.ipynb",
+ "L.stack": "00_torch_core.ipynb",
+ "L.cat": "00_torch_core.ipynb",
+ "concat": "00_torch_core.ipynb",
+ "Chunks": "00_torch_core.ipynb",
+ "show_title": "00_torch_core.ipynb",
+ "ShowTitle": "00_torch_core.ipynb",
+ "TitledInt": "00_torch_core.ipynb",
+ "TitledFloat": "00_torch_core.ipynb",
+ "TitledStr": "00_torch_core.ipynb",
+ "TitledTuple": "00_torch_core.ipynb",
+ "TitledStr.truncate": "00_torch_core.ipynb",
+ "pd.DataFrame.__init__": "00_torch_core.ipynb",
+ "get_empty_df": "00_torch_core.ipynb",
+ "display_df": "00_torch_core.ipynb",
+ "get_first": "00_torch_core.ipynb",
+ "one_param": "00_torch_core.ipynb",
+ "item_find": "00_torch_core.ipynb",
+ "find_device": "00_torch_core.ipynb",
+ "find_bs": "00_torch_core.ipynb",
+ "np_func": "00_torch_core.ipynb",
+ "Module": "00_torch_core.ipynb",
+ "get_model": "00_torch_core.ipynb",
+ "one_hot": "00_torch_core.ipynb",
+ "one_hot_decode": "00_torch_core.ipynb",
+ "params": "00_torch_core.ipynb",
+ "trainable_params": "00_torch_core.ipynb",
+ "norm_types": "00_torch_core.ipynb",
+ "norm_bias_params": "00_torch_core.ipynb",
+ "batch_to_samples": "00_torch_core.ipynb",
+ "Tensor.interp_1d": "00_torch_core.ipynb",
+ "Tensor.pca": "00_torch_core.ipynb",
+ "logit": "00_torch_core.ipynb",
+ "num_distrib": "00_torch_core.ipynb",
+ "rank_distrib": "00_torch_core.ipynb",
+ "distrib_barrier": "00_torch_core.ipynb",
+ "Path.save_array": "00_torch_core.ipynb",
+ "Path.load_array": "00_torch_core.ipynb",
+ "base_doc": "00_torch_core.ipynb",
+ "doc": "00_torch_core.ipynb",
+ "nested_reorder": "00_torch_core.ipynb",
+ "make_cross_image": "00_torch_core.ipynb",
+ "show_image_batch": "00_torch_core.ipynb",
+ "requires_grad": "00_torch_core.ipynb",
+ "init_default": "01_layers.ipynb",
+ "cond_init": "00_torch_core.ipynb",
+ "apply_leaf": "00_torch_core.ipynb",
+ "apply_init": "00_torch_core.ipynb",
+ "script_use_ctx": "00_torch_core.ipynb",
+ "script_save_ctx": "00_torch_core.ipynb",
+ "script_fwd": "00_torch_core.ipynb",
+ "script_bwd": "00_torch_core.ipynb",
+ "grad_module": "00_torch_core.ipynb",
+ "ismin_torch": "00_torch_core.ipynb",
+ "notmax_torch": "00_torch_core.ipynb",
+ "module": "01_layers.ipynb",
+ "Identity": "01_layers.ipynb",
+ "Lambda": "01_layers.ipynb",
+ "PartialLambda": "01_layers.ipynb",
+ "Flatten": "01_layers.ipynb",
+ "ToTensorBase": "01_layers.ipynb",
+ "View": "01_layers.ipynb",
+ "ResizeBatch": "01_layers.ipynb",
+ "Debugger": "01_layers.ipynb",
+ "sigmoid_range": "01_layers.ipynb",
+ "SigmoidRange": "01_layers.ipynb",
+ "AdaptiveConcatPool1d": "01_layers.ipynb",
+ "AdaptiveConcatPool2d": "01_layers.ipynb",
+ "PoolType": "01_layers.ipynb",
+ "adaptive_pool": "01_layers.ipynb",
+ "PoolFlatten": "01_layers.ipynb",
+ "NormType": "01_layers.ipynb",
+ "BatchNorm": "01_layers.ipynb",
+ "InstanceNorm": "01_layers.ipynb",
+ "BatchNorm1dFlat": "01_layers.ipynb",
+ "LinBnDrop": "01_layers.ipynb",
+ "sigmoid": "01_layers.ipynb",
+ "sigmoid_": "01_layers.ipynb",
+ "vleaky_relu": "01_layers.ipynb",
+ "init_linear": "01_layers.ipynb",
+ "defaults.activation": "01_layers.ipynb",
+ "ConvLayer": "01_layers.ipynb",
+ "AdaptiveAvgPool": "01_layers.ipynb",
+ "MaxPool": "01_layers.ipynb",
+ "AvgPool": "01_layers.ipynb",
+ "trunc_normal_": "01_layers.ipynb",
+ "Embedding": "01_layers.ipynb",
+ "SelfAttention": "01_layers.ipynb",
+ "PooledSelfAttention2d": "01_layers.ipynb",
+ "SimpleSelfAttention": "01_layers.ipynb",
+ "icnr_init": "01_layers.ipynb",
+ "PixelShuffle_ICNR": "01_layers.ipynb",
+ "sequential": "01_layers.ipynb",
+ "SequentialEx": "01_layers.ipynb",
+ "MergeLayer": "01_layers.ipynb",
+ "Cat": "01_layers.ipynb",
+ "SimpleCNN": "01_layers.ipynb",
+ "ProdLayer": "01_layers.ipynb",
+ "inplace_relu": "01_layers.ipynb",
+ "SEModule": "01_layers.ipynb",
+ "ResBlock": "01_layers.ipynb",
+ "SEBlock": "01_layers.ipynb",
+ "SEResNeXtBlock": "01_layers.ipynb",
+ "SeparableBlock": "01_layers.ipynb",
+ "TimeDistributed": "01_layers.ipynb",
+ "swish": "01_layers.ipynb",
+ "Swish": "01_layers.ipynb",
+ "MishJitAutoFn": "01_layers.ipynb",
+ "mish": "01_layers.ipynb",
+ "Mish": "01_layers.ipynb",
+ "ParameterModule": "01_layers.ipynb",
+ "children_and_parameters": "01_layers.ipynb",
+ "has_children": "01_layers.ipynb",
+ "flatten_model": "01_layers.ipynb",
+ "NoneReduce": "01_layers.ipynb",
+ "in_channels": "01_layers.ipynb",
+ "BaseLoss": "01a_losses.ipynb",
+ "CrossEntropyLossFlat": "01a_losses.ipynb",
+ "FocalLoss": "01a_losses.ipynb",
+ "FocalLossFlat": "01a_losses.ipynb",
+ "BCEWithLogitsLossFlat": "01a_losses.ipynb",
+ "BCELossFlat": "01a_losses.ipynb",
+ "MSELossFlat": "01a_losses.ipynb",
+ "L1LossFlat": "01a_losses.ipynb",
+ "LabelSmoothingCrossEntropy": "01a_losses.ipynb",
+ "LabelSmoothingCrossEntropyFlat": "01a_losses.ipynb",
+ "DiceLoss": "01a_losses.ipynb",
+ "fa_collate": "02_data.load.ipynb",
+ "fa_convert": "02_data.load.ipynb",
+ "SkipItemException": "02_data.load.ipynb",
+ "collate_error": "02_data.load.ipynb",
+ "DataLoader": "02_data.load.ipynb",
+ "TfmdDL": "03_data.core.ipynb",
+ "DataLoaders": "03_data.core.ipynb",
+ "FilteredBase": "03_data.core.ipynb",
+ "TfmdLists": "03_data.core.ipynb",
+ "decode_at": "03_data.core.ipynb",
+ "show_at": "03_data.core.ipynb",
+ "Datasets": "03_data.core.ipynb",
+ "test_set": "03_data.core.ipynb",
+ "DataLoaders.test_dl": "03_data.core.ipynb",
+ "fastai_cfg": "04_data.external.ipynb",
+ "fastai_path": "04_data.external.ipynb",
+ "URLs": "04_data.external.ipynb",
+ "untar_data": "04_data.external.ipynb",
+ "get_files": "05_data.transforms.ipynb",
+ "FileGetter": "05_data.transforms.ipynb",
+ "image_extensions": "05_data.transforms.ipynb",
+ "get_image_files": "05_data.transforms.ipynb",
+ "ImageGetter": "05_data.transforms.ipynb",
+ "get_text_files": "05_data.transforms.ipynb",
+ "ItemGetter": "05_data.transforms.ipynb",
+ "AttrGetter": "05_data.transforms.ipynb",
+ "RandomSplitter": "05_data.transforms.ipynb",
+ "TrainTestSplitter": "05_data.transforms.ipynb",
+ "IndexSplitter": "05_data.transforms.ipynb",
+ "EndSplitter": "05_data.transforms.ipynb",
+ "GrandparentSplitter": "05_data.transforms.ipynb",
+ "FuncSplitter": "05_data.transforms.ipynb",
+ "MaskSplitter": "05_data.transforms.ipynb",
+ "FileSplitter": "05_data.transforms.ipynb",
+ "ColSplitter": "05_data.transforms.ipynb",
+ "RandomSubsetSplitter": "05_data.transforms.ipynb",
+ "parent_label": "05_data.transforms.ipynb",
+ "RegexLabeller": "05_data.transforms.ipynb",
+ "ColReader": "05_data.transforms.ipynb",
+ "CategoryMap": "05_data.transforms.ipynb",
+ "Categorize": "05_data.transforms.ipynb",
+ "Category": "05_data.transforms.ipynb",
+ "MultiCategorize": "05_data.transforms.ipynb",
+ "MultiCategory": "05_data.transforms.ipynb",
+ "OneHotEncode": "05_data.transforms.ipynb",
+ "EncodedMultiCategorize": "05_data.transforms.ipynb",
+ "RegressionSetup": "05_data.transforms.ipynb",
+ "get_c": "05_data.transforms.ipynb",
+ "ToTensor": "05_data.transforms.ipynb",
+ "IntToFloatTensor": "05_data.transforms.ipynb",
+ "broadcast_vec": "05_data.transforms.ipynb",
+ "Normalize": "05_data.transforms.ipynb",
+ "TransformBlock": "06_data.block.ipynb",
+ "CategoryBlock": "06_data.block.ipynb",
+ "MultiCategoryBlock": "06_data.block.ipynb",
+ "RegressionBlock": "06_data.block.ipynb",
+ "DataBlock": "06_data.block.ipynb",
+ "DataBlock.summary": "06_data.block.ipynb",
+ "imagenet_stats": "07_vision.core.ipynb",
+ "cifar_stats": "07_vision.core.ipynb",
+ "mnist_stats": "07_vision.core.ipynb",
+ "n_px": "07_vision.core.ipynb",
+ "shape": "60_medical.imaging.ipynb",
+ "aspect": "07_vision.core.ipynb",
+ "Image.Image.reshape": "07_vision.core.ipynb",
+ "Image.Image.to_bytes_format": "07_vision.core.ipynb",
+ "Image.Image.to_thumb": "07_vision.core.ipynb",
+ "Image.Image.resize_max": "07_vision.core.ipynb",
+ "to_image": "07_vision.core.ipynb",
+ "load_image": "07_vision.core.ipynb",
+ "image2tensor": "07_vision.core.ipynb",
+ "PILBase": "07_vision.core.ipynb",
+ "PILImage": "07_vision.core.ipynb",
+ "PILImageBW": "07_vision.core.ipynb",
+ "PILMask": "07_vision.core.ipynb",
+ "OpenMask": "07_vision.core.ipynb",
+ "OpenMask.loss_func": "07_vision.core.ipynb",
+ "PILMask.create": "07_vision.core.ipynb",
+ "AddMaskCodes": "07_vision.core.ipynb",
+ "TensorPoint": "07_vision.core.ipynb",
+ "TensorPointCreate": "07_vision.core.ipynb",
+ "TensorPointCreate.loss_func": "07_vision.core.ipynb",
+ "TensorPoint.create": "07_vision.core.ipynb",
+ "get_annotations": "07_vision.core.ipynb",
+ "TensorBBox": "07_vision.core.ipynb",
+ "LabeledBBox": "07_vision.core.ipynb",
+ "encodes": "40_tabular.core.ipynb",
+ "PointScaler": "07_vision.core.ipynb",
+ "BBoxLabeler": "07_vision.core.ipynb",
+ "decodes": "40_tabular.core.ipynb",
+ "get_grid": "08_vision.data.ipynb",
+ "clip_remove_empty": "08_vision.data.ipynb",
+ "bb_pad": "08_vision.data.ipynb",
+ "ImageBlock": "08_vision.data.ipynb",
+ "MaskBlock": "08_vision.data.ipynb",
+ "PointBlock": "08_vision.data.ipynb",
+ "BBoxBlock": "08_vision.data.ipynb",
+ "PointBlock.__doc__": "08_vision.data.ipynb",
+ "BBoxBlock.__doc__": "08_vision.data.ipynb",
+ "BBoxLblBlock": "08_vision.data.ipynb",
+ "ImageDataLoaders": "08_vision.data.ipynb",
+ "ImageDataLoaders.from_csv": "08_vision.data.ipynb",
+ "ImageDataLoaders.from_name_func": "08_vision.data.ipynb",
+ "ImageDataLoaders.from_path_re": "08_vision.data.ipynb",
+ "ImageDataLoaders.from_name_re": "08_vision.data.ipynb",
+ "SegmentationDataLoaders": "08_vision.data.ipynb",
+ "RandTransform": "09_vision.augment.ipynb",
+ "TensorTypes": "09_vision.augment.ipynb",
+ "Image.Image.flip_lr": "09_vision.augment.ipynb",
+ "TensorImageBase.flip_lr": "09_vision.augment.ipynb",
+ "TensorPoint.flip_lr": "09_vision.augment.ipynb",
+ "TensorBBox.flip_lr": "09_vision.augment.ipynb",
+ "FlipItem": "09_vision.augment.ipynb",
+ "PILImage.dihedral": "09_vision.augment.ipynb",
+ "TensorImage.dihedral": "09_vision.augment.ipynb",
+ "TensorPoint.dihedral": "09_vision.augment.ipynb",
+ "TensorBBox.dihedral": "09_vision.augment.ipynb",
+ "DihedralItem": "09_vision.augment.ipynb",
+ "TensorBBox.crop_pad": "09_vision.augment.ipynb",
+ "TensorPoint.crop_pad": "09_vision.augment.ipynb",
+ "Image.Image.crop_pad": "09_vision.augment.ipynb",
+ "CropPad": "09_vision.augment.ipynb",
+ "RandomCrop": "09_vision.augment.ipynb",
+ "OldRandomCrop": "09_vision.augment.ipynb",
+ "Resize": "09_vision.augment.ipynb",
+ "RandomResizedCrop": "09_vision.augment.ipynb",
+ "RatioResize": "09_vision.augment.ipynb",
+ "affine_grid": "09_vision.augment.ipynb",
+ "TensorImage.affine_coord": "09_vision.augment.ipynb",
+ "TensorMask.affine_coord": "09_vision.augment.ipynb",
+ "TensorPoint.affine_coord": "09_vision.augment.ipynb",
+ "TensorBBox.affine_coord": "09_vision.augment.ipynb",
+ "AffineCoordTfm": "09_vision.augment.ipynb",
+ "RandomResizedCropGPU": "09_vision.augment.ipynb",
+ "mask_tensor": "09_vision.augment.ipynb",
+ "affine_mat": "09_vision.augment.ipynb",
+ "flip_mat": "09_vision.augment.ipynb",
+ "TensorImage.flip_batch": "09_vision.augment.ipynb",
+ "TensorMask.flip_batch": "09_vision.augment.ipynb",
+ "TensorPoint.flip_batch": "09_vision.augment.ipynb",
+ "TensorBBox.flip_batch": "09_vision.augment.ipynb",
+ "Flip": "09_vision.augment.ipynb",
+ "DeterministicDraw": "09_vision.augment.ipynb",
+ "DeterministicFlip": "09_vision.augment.ipynb",
+ "dihedral_mat": "09_vision.augment.ipynb",
+ "TensorImage.dihedral_batch": "09_vision.augment.ipynb",
+ "TensorMask.dihedral_batch": "09_vision.augment.ipynb",
+ "TensorPoint.dihedral_batch": "09_vision.augment.ipynb",
+ "TensorBBox.dihedral_batch": "09_vision.augment.ipynb",
+ "Dihedral": "09_vision.augment.ipynb",
+ "DeterministicDihedral": "09_vision.augment.ipynb",
+ "rotate_mat": "09_vision.augment.ipynb",
+ "TensorImage.rotate": "09_vision.augment.ipynb",
+ "TensorMask.rotate": "09_vision.augment.ipynb",
+ "TensorPoint.rotate": "09_vision.augment.ipynb",
+ "TensorBBox.rotate": "09_vision.augment.ipynb",
+ "Rotate": "09_vision.augment.ipynb",
+ "zoom_mat": "09_vision.augment.ipynb",
+ "TensorImage.zoom": "09_vision.augment.ipynb",
+ "TensorMask.zoom": "09_vision.augment.ipynb",
+ "TensorPoint.zoom": "09_vision.augment.ipynb",
+ "TensorBBox.zoom": "09_vision.augment.ipynb",
+ "Zoom": "09_vision.augment.ipynb",
+ "find_coeffs": "09_vision.augment.ipynb",
+ "apply_perspective": "09_vision.augment.ipynb",
+ "TensorImage.warp": "09_vision.augment.ipynb",
+ "TensorMask.warp": "09_vision.augment.ipynb",
+ "TensorPoint.warp": "09_vision.augment.ipynb",
+ "TensorBBox.warp": "09_vision.augment.ipynb",
+ "Warp": "09_vision.augment.ipynb",
+ "TensorImage.lighting": "09_vision.augment.ipynb",
+ "SpaceTfm": "09_vision.augment.ipynb",
+ "LightingTfm": "09_vision.augment.ipynb",
+ "TensorImage.brightness": "09_vision.augment.ipynb",
+ "Brightness": "09_vision.augment.ipynb",
+ "TensorImage.contrast": "09_vision.augment.ipynb",
+ "Contrast": "09_vision.augment.ipynb",
+ "grayscale": "09_vision.augment.ipynb",
+ "TensorImage.saturation": "09_vision.augment.ipynb",
+ "Saturation": "09_vision.augment.ipynb",
+ "rgb2hsv": "09_vision.augment.ipynb",
+ "hsv2rgb": "09_vision.augment.ipynb",
+ "TensorImage.hsv": "09_vision.augment.ipynb",
+ "HSVTfm": "09_vision.augment.ipynb",
+ "TensorImage.hue": "09_vision.augment.ipynb",
+ "Hue": "09_vision.augment.ipynb",
+ "cutout_gaussian": "09_vision.augment.ipynb",
+ "norm_apply_denorm": "09_vision.augment.ipynb",
+ "RandomErasing": "09_vision.augment.ipynb",
+ "setup_aug_tfms": "09_vision.augment.ipynb",
+ "aug_transforms": "09_vision.augment.ipynb",
+ "download_images": "09b_vision.utils.ipynb",
+ "resize_to": "09b_vision.utils.ipynb",
+ "verify_image": "09b_vision.utils.ipynb",
+ "verify_images": "09b_vision.utils.ipynb",
+ "resize_image": "09b_vision.utils.ipynb",
+ "resize_images": "09b_vision.utils.ipynb",
+ "Box.__getitem__": "09c_vision.widgets.ipynb",
+ "widget": "09c_vision.widgets.ipynb",
+ "carousel": "09c_vision.widgets.ipynb",
+ "ImagesCleaner": "09c_vision.widgets.ipynb",
+ "ImageClassifierCleaner": "09c_vision.widgets.ipynb",
+ "init_cnn": "11_vision.models.xresnet.ipynb",
+ "XResNet": "11_vision.models.xresnet.ipynb",
+ "xresnet18": "11_vision.models.xresnet.ipynb",
+ "xresnet34": "11_vision.models.xresnet.ipynb",
+ "xresnet50": "11_vision.models.xresnet.ipynb",
+ "xresnet101": "11_vision.models.xresnet.ipynb",
+ "xresnet152": "11_vision.models.xresnet.ipynb",
+ "xresnet18_deep": "11_vision.models.xresnet.ipynb",
+ "xresnet34_deep": "11_vision.models.xresnet.ipynb",
+ "xresnet50_deep": "11_vision.models.xresnet.ipynb",
+ "xresnet18_deeper": "11_vision.models.xresnet.ipynb",
+ "xresnet34_deeper": "11_vision.models.xresnet.ipynb",
+ "xresnet50_deeper": "11_vision.models.xresnet.ipynb",
+ "se_kwargs1": "11_vision.models.xresnet.ipynb",
+ "se_kwargs2": "11_vision.models.xresnet.ipynb",
+ "se_kwargs3": "11_vision.models.xresnet.ipynb",
+ "g0": "11_vision.models.xresnet.ipynb",
+ "g1": "11_vision.models.xresnet.ipynb",
+ "g2": "11_vision.models.xresnet.ipynb",
+ "g3": "11_vision.models.xresnet.ipynb",
+ "xse_resnet18": "11_vision.models.xresnet.ipynb",
+ "xse_resnext18": "11_vision.models.xresnet.ipynb",
+ "xresnext18": "11_vision.models.xresnet.ipynb",
+ "xse_resnet34": "11_vision.models.xresnet.ipynb",
+ "xse_resnext34": "11_vision.models.xresnet.ipynb",
+ "xresnext34": "11_vision.models.xresnet.ipynb",
+ "xse_resnet50": "11_vision.models.xresnet.ipynb",
+ "xse_resnext50": "11_vision.models.xresnet.ipynb",
+ "xresnext50": "11_vision.models.xresnet.ipynb",
+ "xse_resnet101": "11_vision.models.xresnet.ipynb",
+ "xse_resnext101": "11_vision.models.xresnet.ipynb",
+ "xresnext101": "11_vision.models.xresnet.ipynb",
+ "xse_resnet152": "11_vision.models.xresnet.ipynb",
+ "xsenet154": "11_vision.models.xresnet.ipynb",
+ "xse_resnext18_deep": "11_vision.models.xresnet.ipynb",
+ "xse_resnext34_deep": "11_vision.models.xresnet.ipynb",
+ "xse_resnext50_deep": "11_vision.models.xresnet.ipynb",
+ "xse_resnext18_deeper": "11_vision.models.xresnet.ipynb",
+ "xse_resnext34_deeper": "11_vision.models.xresnet.ipynb",
+ "xse_resnext50_deeper": "11_vision.models.xresnet.ipynb",
+ "Optimizer": "12_optimizer.ipynb",
+ "sgd_step": "12_optimizer.ipynb",
+ "weight_decay": "12_optimizer.ipynb",
+ "weight_decay.defaults": "12_optimizer.ipynb",
+ "l2_reg": "12_optimizer.ipynb",
+ "l2_reg.defaults": "12_optimizer.ipynb",
+ "average_grad": "12_optimizer.ipynb",
+ "average_grad.defaults": "12_optimizer.ipynb",
+ "average_sqr_grad": "12_optimizer.ipynb",
+ "average_sqr_grad.defaults": "12_optimizer.ipynb",
+ "momentum_step": "12_optimizer.ipynb",
+ "SGD": "12_optimizer.ipynb",
+ "rms_prop_step": "12_optimizer.ipynb",
+ "rms_prop_step.defaults": "12_optimizer.ipynb",
+ "RMSProp": "12_optimizer.ipynb",
+ "step_stat": "12_optimizer.ipynb",
+ "debias": "12_optimizer.ipynb",
+ "adam_step": "12_optimizer.ipynb",
+ "Adam": "12_optimizer.ipynb",
+ "radam_step": "12_optimizer.ipynb",
+ "RAdam": "12_optimizer.ipynb",
+ "qhadam_step": "12_optimizer.ipynb",
+ "QHAdam": "12_optimizer.ipynb",
+ "larc_layer_lr": "12_optimizer.ipynb",
+ "larc_layer_lr.defaults": "12_optimizer.ipynb",
+ "larc_step": "12_optimizer.ipynb",
+ "Larc": "12_optimizer.ipynb",
+ "lamb_step": "12_optimizer.ipynb",
+ "Lamb": "12_optimizer.ipynb",
+ "Lookahead": "12_optimizer.ipynb",
+ "ranger": "12_optimizer.ipynb",
+ "detuplify_pg": "12_optimizer.ipynb",
+ "set_item_pg": "12_optimizer.ipynb",
+ "pytorch_hp_map": "12_optimizer.ipynb",
+ "OptimWrapper": "12_optimizer.ipynb",
+ "Callback": "13_callback.core.ipynb",
+ "TrainEvalCallback": "13_callback.core.ipynb",
+ "GatherPredsCallback": "13_callback.core.ipynb",
+ "FetchPredsCallback": "13_callback.core.ipynb",
+ "defaults.lr": "13a_learner.ipynb",
+ "replacing_yield": "13a_learner.ipynb",
+ "mk_metric": "13a_learner.ipynb",
+ "save_model": "13a_learner.ipynb",
+ "load_model": "13a_learner.ipynb",
+ "SkipToEpoch": "13a_learner.ipynb",
+ "Learner": "13a_learner.ipynb",
+ "before_batch_cb": "13a_learner.ipynb",
+ "Learner.save": "13a_learner.ipynb",
+ "Learner.load": "13a_learner.ipynb",
+ "Learner.export": "13a_learner.ipynb",
+ "load_learner": "13a_learner.ipynb",
+ "Metric": "13a_learner.ipynb",
+ "AvgMetric": "13a_learner.ipynb",
+ "AvgLoss": "13a_learner.ipynb",
+ "AvgSmoothLoss": "13a_learner.ipynb",
+ "ValueMetric": "13a_learner.ipynb",
+ "Recorder": "13a_learner.ipynb",
+ "CastToTensor": "13a_learner.ipynb",
+ "Learner.freeze_to": "13a_learner.ipynb",
+ "Learner.freeze": "13a_learner.ipynb",
+ "Learner.unfreeze": "13a_learner.ipynb",
+ "Learner.tta": "13a_learner.ipynb",
+ "flatten_check": "13b_metrics.ipynb",
+ "AccumMetric": "13b_metrics.ipynb",
+ "skm_to_fastai": "13b_metrics.ipynb",
+ "optim_metric": "13b_metrics.ipynb",
+ "accuracy": "13b_metrics.ipynb",
+ "error_rate": "13b_metrics.ipynb",
+ "top_k_accuracy": "13b_metrics.ipynb",
+ "APScoreBinary": "13b_metrics.ipynb",
+ "BalancedAccuracy": "13b_metrics.ipynb",
+ "BrierScore": "13b_metrics.ipynb",
+ "CohenKappa": "13b_metrics.ipynb",
+ "F1Score": "13b_metrics.ipynb",
+ "FBeta": "13b_metrics.ipynb",
+ "HammingLoss": "13b_metrics.ipynb",
+ "Jaccard": "13b_metrics.ipynb",
+ "Precision": "13b_metrics.ipynb",
+ "Recall": "13b_metrics.ipynb",
+ "RocAuc": "13b_metrics.ipynb",
+ "RocAucBinary": "13b_metrics.ipynb",
+ "MatthewsCorrCoef": "13b_metrics.ipynb",
+ "accuracy_multi": "13b_metrics.ipynb",
+ "APScoreMulti": "13b_metrics.ipynb",
+ "BrierScoreMulti": "13b_metrics.ipynb",
+ "F1ScoreMulti": "13b_metrics.ipynb",
+ "FBetaMulti": "13b_metrics.ipynb",
+ "HammingLossMulti": "13b_metrics.ipynb",
+ "JaccardMulti": "13b_metrics.ipynb",
+ "MatthewsCorrCoefMulti": "13b_metrics.ipynb",
+ "PrecisionMulti": "13b_metrics.ipynb",
+ "RecallMulti": "13b_metrics.ipynb",
+ "RocAucMulti": "13b_metrics.ipynb",
+ "mse": "13b_metrics.ipynb",
+ "rmse": "13b_metrics.ipynb",
+ "rmse.__doc__": "13b_metrics.ipynb",
+ "mae": "13b_metrics.ipynb",
+ "msle": "13b_metrics.ipynb",
+ "exp_rmspe": "13b_metrics.ipynb",
+ "exp_rmspe.__doc__": "13b_metrics.ipynb",
+ "ExplainedVariance": "13b_metrics.ipynb",
+ "R2Score": "13b_metrics.ipynb",
+ "PearsonCorrCoef": "13b_metrics.ipynb",
+ "SpearmanCorrCoef": "13b_metrics.ipynb",
+ "foreground_acc": "13b_metrics.ipynb",
+ "Dice": "13b_metrics.ipynb",
+ "DiceMulti": "13b_metrics.ipynb",
+ "JaccardCoeff": "13b_metrics.ipynb",
+ "CorpusBLEUMetric": "13b_metrics.ipynb",
+ "Perplexity": "13b_metrics.ipynb",
+ "perplexity": "13b_metrics.ipynb",
+ "LossMetric": "13b_metrics.ipynb",
+ "LossMetrics": "13b_metrics.ipynb",
+ "annealer": "14_callback.schedule.ipynb",
+ "sched_lin": "14_callback.schedule.ipynb",
+ "sched_cos": "14_callback.schedule.ipynb",
+ "sched_no": "14_callback.schedule.ipynb",
+ "sched_exp": "14_callback.schedule.ipynb",
+ "SchedLin": "14_callback.schedule.ipynb",
+ "SchedCos": "14_callback.schedule.ipynb",
+ "SchedNo": "14_callback.schedule.ipynb",
+ "SchedExp": "14_callback.schedule.ipynb",
+ "SchedLin.__doc__": "14_callback.schedule.ipynb",
+ "SchedCos.__doc__": "14_callback.schedule.ipynb",
+ "SchedExp.__doc__": "14_callback.schedule.ipynb",
+ "SchedPoly": "14_callback.schedule.ipynb",
+ "combine_scheds": "14_callback.schedule.ipynb",
+ "combined_cos": "14_callback.schedule.ipynb",
+ "ParamScheduler": "14_callback.schedule.ipynb",
+ "Learner.fit_one_cycle": "14_callback.schedule.ipynb",
+ "Recorder.plot_sched": "14_callback.schedule.ipynb",
+ "Learner.fit_flat_cos": "14_callback.schedule.ipynb",
+ "Learner.fit_sgdr": "14_callback.schedule.ipynb",
+ "Learner.fine_tune": "14_callback.schedule.ipynb",
+ "LRFinder": "14_callback.schedule.ipynb",
+ "valley": "14_callback.schedule.ipynb",
+ "slide": "14_callback.schedule.ipynb",
+ "minimum": "14_callback.schedule.ipynb",
+ "steep": "14_callback.schedule.ipynb",
+ "Recorder.plot_lr_find": "14_callback.schedule.ipynb",
+ "Learner.lr_find": "14_callback.schedule.ipynb",
+ "CollectDataCallback": "14a_callback.data.ipynb",
+ "WeightedDL": "14a_callback.data.ipynb",
+ "Datasets.weighted_dataloaders": "14a_callback.data.ipynb",
+ "DataBlock.weighted_dataloaders": "14a_callback.data.ipynb",
+ "PartialDL": "14a_callback.data.ipynb",
+ "FilteredBase.partial_dataloaders": "14a_callback.data.ipynb",
+ "Hook": "15_callback.hook.ipynb",
+ "hook_output": "15_callback.hook.ipynb",
+ "Hooks": "15_callback.hook.ipynb",
+ "hook_outputs": "15_callback.hook.ipynb",
+ "dummy_eval": "15_callback.hook.ipynb",
+ "model_sizes": "15_callback.hook.ipynb",
+ "num_features_model": "15_callback.hook.ipynb",
+ "has_params": "15_callback.hook.ipynb",
+ "HookCallback": "15_callback.hook.ipynb",
+ "total_params": "15_callback.hook.ipynb",
+ "layer_info": "15_callback.hook.ipynb",
+ "module_summary": "15_callback.hook.ipynb",
+ "Learner.summary": "15_callback.hook.ipynb",
+ "ActivationStats": "15_callback.hook.ipynb",
+ "UnetBlock": "15a_vision.models.unet.ipynb",
+ "ResizeToOrig": "15a_vision.models.unet.ipynb",
+ "DynamicUnet": "15a_vision.models.unet.ipynb",
+ "ProgressCallback": "16_callback.progress.ipynb",
+ "Learner.no_bar": "16_callback.progress.ipynb",
+ "ShowGraphCallback": "16_callback.progress.ipynb",
+ "CSVLogger": "16_callback.progress.ipynb",
+ "TerminateOnNaNCallback": "17_callback.tracker.ipynb",
+ "TrackerCallback": "17_callback.tracker.ipynb",
+ "EarlyStoppingCallback": "17_callback.tracker.ipynb",
+ "SaveModelCallback": "17_callback.tracker.ipynb",
+ "ReduceLROnPlateau": "17_callback.tracker.ipynb",
+ "MixedPrecision": "18_callback.fp16.ipynb",
+ "FP16TestCallback": "18_callback.fp16.ipynb",
+ "Learner.to_fp16": "18_callback.fp16.ipynb",
+ "Learner.to_fp32": "18_callback.fp16.ipynb",
+ "get_master": "18_callback.fp16.ipynb",
+ "to_master_grads": "18_callback.fp16.ipynb",
+ "to_model_params": "18_callback.fp16.ipynb",
+ "test_overflow": "18_callback.fp16.ipynb",
+ "grad_overflow": "18_callback.fp16.ipynb",
+ "copy_clone": "18_callback.fp16.ipynb",
+ "ModelToHalf": "18_callback.fp16.ipynb",
+ "NonNativeMixedPrecision": "18_callback.fp16.ipynb",
+ "Learner.to_non_native_fp16": "18_callback.fp16.ipynb",
+ "Learner.to_non_native_fp32": "18_callback.fp16.ipynb",
+ "ShortEpochCallback": "18a_callback.training.ipynb",
+ "GradientAccumulation": "18a_callback.training.ipynb",
+ "GradientClip": "18a_callback.training.ipynb",
+ "set_bn_eval": "18a_callback.training.ipynb",
+ "BnFreeze": "18a_callback.training.ipynb",
+ "bn_types": "18a_callback.training.ipynb",
+ "ChannelsLast": "18a_callback.training.ipynb",
+ "MCDropoutCallback": "18b_callback.preds.ipynb",
+ "reduce_loss": "19_callback.mixup.ipynb",
+ "MixHandler": "19_callback.mixup.ipynb",
+ "MixUp": "19_callback.mixup.ipynb",
+ "CutMix": "19_callback.mixup.ipynb",
+ "Interpretation": "20_interpret.ipynb",
+ "ClassificationInterpretation": "20_interpret.ipynb",
+ "SegmentationInterpretation": "20_interpret.ipynb",
+ "DataParallel.reset": "20a_distributed.ipynb",
+ "ParallelTrainer": "20a_distributed.ipynb",
+ "Learner.to_parallel": "20a_distributed.ipynb",
+ "Learner.detach_parallel": "20a_distributed.ipynb",
+ "Learner.parallel_ctx": "20a_distributed.ipynb",
+ "DistributedDataParallel.reset": "20a_distributed.ipynb",
+ "setup_distrib": "20a_distributed.ipynb",
+ "teardown_distrib": "20a_distributed.ipynb",
+ "DistributedDL": "20a_distributed.ipynb",
+ "DistributedTrainer": "20a_distributed.ipynb",
+ "Learner.to_distributed": "20a_distributed.ipynb",
+ "Learner.detach_distributed": "20a_distributed.ipynb",
+ "Learner.distrib_ctx": "20a_distributed.ipynb",
+ "rank0_first": "20a_distributed.ipynb",
+ "has_pool_type": "21_vision.learner.ipynb",
+ "cut_model": "21_vision.learner.ipynb",
+ "create_body": "21_vision.learner.ipynb",
+ "create_head": "21_vision.learner.ipynb",
+ "default_split": "21_vision.learner.ipynb",
+ "model_meta": "21_vision.learner.ipynb",
+ "add_head": "21_vision.learner.ipynb",
+ "create_vision_model": "21_vision.learner.ipynb",
+ "TimmBody": "21_vision.learner.ipynb",
+ "create_timm_model": "21_vision.learner.ipynb",
+ "vision_learner": "21_vision.learner.ipynb",
+ "create_unet_model": "21_vision.learner.ipynb",
+ "unet_learner": "21_vision.learner.ipynb",
+ "create_cnn_model": "21_vision.learner.ipynb",
+ "cnn_learner": "21_vision.learner.ipynb",
+ "GANModule": "24_vision.gan.ipynb",
+ "basic_critic": "24_vision.gan.ipynb",
+ "AddChannels": "24_vision.gan.ipynb",
+ "basic_generator": "24_vision.gan.ipynb",
+ "DenseResBlock": "24_vision.gan.ipynb",
+ "gan_critic": "24_vision.gan.ipynb",
+ "GANLoss": "24_vision.gan.ipynb",
+ "AdaptiveLoss": "24_vision.gan.ipynb",
+ "accuracy_thresh_expand": "24_vision.gan.ipynb",
+ "set_freeze_model": "24_vision.gan.ipynb",
+ "GANTrainer": "24_vision.gan.ipynb",
+ "FixedGANSwitcher": "24_vision.gan.ipynb",
+ "AdaptiveGANSwitcher": "24_vision.gan.ipynb",
+ "GANDiscriminativeLR": "24_vision.gan.ipynb",
+ "InvisibleTensor": "24_vision.gan.ipynb",
+ "generate_noise": "24_vision.gan.ipynb",
+ "gan_loss_from_func": "24_vision.gan.ipynb",
+ "GANLearner": "24_vision.gan.ipynb",
+ "GANLearner.from_learners": "24_vision.gan.ipynb",
+ "GANLearner.wgan": "24_vision.gan.ipynb",
+ "spec_add_spaces": "30_text.core.ipynb",
+ "rm_useless_spaces": "30_text.core.ipynb",
+ "replace_rep": "30_text.core.ipynb",
+ "replace_wrep": "30_text.core.ipynb",
+ "fix_html": "30_text.core.ipynb",
+ "replace_all_caps": "30_text.core.ipynb",
+ "replace_maj": "30_text.core.ipynb",
+ "lowercase": "30_text.core.ipynb",
+ "replace_space": "30_text.core.ipynb",
+ "defaults.text_spec_tok": "30_text.core.ipynb",
+ "defaults.text_proc_rules": "30_text.core.ipynb",
+ "defaults.text_postproc_rules": "30_text.core.ipynb",
+ "BaseTokenizer": "30_text.core.ipynb",
+ "SpacyTokenizer": "30_text.core.ipynb",
+ "WordTokenizer": "30_text.core.ipynb",
+ "TokenizeWithRules": "30_text.core.ipynb",
+ "tokenize1": "30_text.core.ipynb",
+ "parallel_tokenize": "30_text.core.ipynb",
+ "fn_counter_pkl": "30_text.core.ipynb",
+ "fn_lengths_pkl": "30_text.core.ipynb",
+ "tokenize_folder": "30_text.core.ipynb",
+ "tokenize_files": "30_text.core.ipynb",
+ "tokenize_texts": "30_text.core.ipynb",
+ "tokenize_df": "30_text.core.ipynb",
+ "tokenize_csv": "30_text.core.ipynb",
+ "load_tokenized_csv": "30_text.core.ipynb",
+ "Tokenizer": "30_text.core.ipynb",
+ "eu_langs": "30_text.core.ipynb",
+ "SentencePieceTokenizer": "30_text.core.ipynb",
+ "SubwordTokenizer": "30_text.core.ipynb",
+ "reverse_text": "31_text.data.ipynb",
+ "make_vocab": "31_text.data.ipynb",
+ "TensorText": "31_text.data.ipynb",
+ "LMTensorText": "31_text.data.ipynb",
+ "TensorText.__doc__": "31_text.data.ipynb",
+ "LMTensorText.__doc__": "31_text.data.ipynb",
+ "Numericalize": "31_text.data.ipynb",
+ "LMDataLoader": "31_text.data.ipynb",
+ "Pad_Input": "31_text.data.ipynb",
+ "pad_input": "31_text.data.ipynb",
+ "pad_chunk": "31_text.data.ipynb",
+ "pad_input_chunk": "31_text.data.ipynb",
+ "Pad_Chunk": "31_text.data.ipynb",
+ "SortedDL": "31_text.data.ipynb",
+ "TextBlock": "31_text.data.ipynb",
+ "TextDataLoaders": "31_text.data.ipynb",
+ "TextDataLoaders.from_csv": "31_text.data.ipynb",
+ "dropout_mask": "32_text.models.awdlstm.ipynb",
+ "RNNDropout": "32_text.models.awdlstm.ipynb",
+ "WeightDropout": "32_text.models.awdlstm.ipynb",
+ "EmbeddingDropout": "32_text.models.awdlstm.ipynb",
+ "AWD_LSTM": "32_text.models.awdlstm.ipynb",
+ "awd_lstm_lm_split": "32_text.models.awdlstm.ipynb",
+ "awd_lstm_lm_config": "32_text.models.awdlstm.ipynb",
+ "awd_lstm_clas_split": "32_text.models.awdlstm.ipynb",
+ "awd_lstm_clas_config": "32_text.models.awdlstm.ipynb",
+ "LinearDecoder": "33_text.models.core.ipynb",
+ "SequentialRNN": "33_text.models.core.ipynb",
+ "get_language_model": "33_text.models.core.ipynb",
+ "SentenceEncoder": "33_text.models.core.ipynb",
+ "masked_concat_pool": "33_text.models.core.ipynb",
+ "PoolingLinearClassifier": "33_text.models.core.ipynb",
+ "get_text_classifier": "33_text.models.core.ipynb",
+ "ModelResetter": "34_callback.rnn.ipynb",
+ "RNNCallback": "34_callback.rnn.ipynb",
+ "RNNRegularizer": "34_callback.rnn.ipynb",
+ "rnn_cbs": "34_callback.rnn.ipynb",
+ "match_embeds": "37_text.learner.ipynb",
+ "load_ignore_keys": "37_text.learner.ipynb",
+ "clean_raw_keys": "37_text.learner.ipynb",
+ "load_model_text": "37_text.learner.ipynb",
+ "TextLearner": "37_text.learner.ipynb",
+ "decode_spec_tokens": "37_text.learner.ipynb",
+ "LMLearner": "37_text.learner.ipynb",
+ "language_model_learner": "37_text.learner.ipynb",
+ "text_classifier_learner": "37_text.learner.ipynb",
+ "make_date": "40_tabular.core.ipynb",
+ "add_datepart": "40_tabular.core.ipynb",
+ "add_elapsed_times": "40_tabular.core.ipynb",
+ "cont_cat_split": "40_tabular.core.ipynb",
+ "df_shrink_dtypes": "40_tabular.core.ipynb",
+ "df_shrink": "40_tabular.core.ipynb",
+ "Tabular": "40_tabular.core.ipynb",
+ "TabularPandas": "40_tabular.core.ipynb",
+ "TabularProc": "40_tabular.core.ipynb",
+ "Categorify": "40_tabular.core.ipynb",
+ "setups": "40_tabular.core.ipynb",
+ "FillStrategy": "40_tabular.core.ipynb",
+ "FillMissing": "40_tabular.core.ipynb",
+ "ReadTabBatch": "40_tabular.core.ipynb",
+ "TabDataLoader": "40_tabular.core.ipynb",
+ "TabularDataLoaders": "41_tabular.data.ipynb",
+ "TabularDataLoaders.from_csv": "41_tabular.data.ipynb",
+ "emb_sz_rule": "42_tabular.model.ipynb",
+ "get_emb_sz": "42_tabular.model.ipynb",
+ "TabularModel": "42_tabular.model.ipynb",
+ "tabular_config": "42_tabular.model.ipynb",
+ "TabularLearner": "43_tabular.learner.ipynb",
+ "tabular_learner": "43_tabular.learner.ipynb",
+ "TabularCollab": "45_collab.ipynb",
+ "CollabDataLoaders": "45_collab.ipynb",
+ "CollabDataLoaders.from_csv": "45_collab.ipynb",
+ "EmbeddingDotBias": "45_collab.ipynb",
+ "EmbeddingNN": "45_collab.ipynb",
+ "collab_learner": "45_collab.ipynb",
+ "get_dicom_files": "60_medical.imaging.ipynb",
+ "Path.dcmread": "60_medical.imaging.ipynb",
+ "TensorDicom": "60_medical.imaging.ipynb",
+ "PILDicom": "60_medical.imaging.ipynb",
+ "Path.png16read": "60_medical.imaging.ipynb",
+ "pixels": "60_medical.imaging.ipynb",
+ "scaled_px": "60_medical.imaging.ipynb",
+ "array_freqhist_bins": "60_medical.imaging.ipynb",
+ "Tensor.freqhist_bins": "60_medical.imaging.ipynb",
+ "Tensor.hist_scaled_pt": "60_medical.imaging.ipynb",
+ "Tensor.hist_scaled": "60_medical.imaging.ipynb",
+ "DcmDataset.hist_scaled": "60_medical.imaging.ipynb",
+ "Tensor.windowed": "60_medical.imaging.ipynb",
+ "DcmDataset.windowed": "60_medical.imaging.ipynb",
+ "dicom_windows": "60_medical.imaging.ipynb",
+ "TensorCTScan": "60_medical.imaging.ipynb",
+ "PILCTScan": "60_medical.imaging.ipynb",
+ "DcmDataset.show": "60_medical.imaging.ipynb",
+ "DcmDataset.pct_in_window": "60_medical.imaging.ipynb",
+ "uniform_blur2d": "60_medical.imaging.ipynb",
+ "gauss_blur2d": "60_medical.imaging.ipynb",
+ "Tensor.mask_from_blur": "60_medical.imaging.ipynb",
+ "DcmDataset.mask_from_blur": "60_medical.imaging.ipynb",
+ "mask2bbox": "60_medical.imaging.ipynb",
+ "crop_resize": "60_medical.imaging.ipynb",
+ "Tensor.to_nchan": "60_medical.imaging.ipynb",
+ "DcmDataset.to_nchan": "60_medical.imaging.ipynb",
+ "Tensor.to_3chan": "60_medical.imaging.ipynb",
+ "DcmDataset.to_3chan": "60_medical.imaging.ipynb",
+ "Tensor.save_jpg": "60_medical.imaging.ipynb",
+ "DcmDataset.save_jpg": "60_medical.imaging.ipynb",
+ "Tensor.to_uint16": "60_medical.imaging.ipynb",
+ "DcmDataset.to_uint16": "60_medical.imaging.ipynb",
+ "Tensor.save_tif16": "60_medical.imaging.ipynb",
+ "DcmDataset.save_tif16": "60_medical.imaging.ipynb",
+ "DcmDataset.set_pixels": "60_medical.imaging.ipynb",
+ "DcmDataset.pixel_array": "60_medical.imaging.ipynb",
+ "DcmDataset.zoom": "60_medical.imaging.ipynb",
+ "DcmDataset.zoom_to": "60_medical.imaging.ipynb",
+ "DcmDataset.as_dict": "60_medical.imaging.ipynb",
+ "pd.DataFrame.from_dicoms": "60_medical.imaging.ipynb",
+ "DicomSegmentationDataLoaders": "60_medical.imaging.ipynb",
+ "WandbCallback": "70_callback.wandb.ipynb",
+ "Learner.gather_args": "70_callback.wandb.ipynb",
+ "log_dataset": "70_callback.wandb.ipynb",
+ "log_model": "70_callback.wandb.ipynb",
+ "TensorBoardBaseCallback": "70a_callback.tensorboard.ipynb",
+ "TensorBoardCallback": "70a_callback.tensorboard.ipynb",
+ "TensorBoardProjectorCallback": "70a_callback.tensorboard.ipynb",
+ "projector_word_embeddings": "70a_callback.tensorboard.ipynb",
+ "NeptuneCallback": "70b_callback.neptune.ipynb",
+ "json_clean": "70c_callback.captum.ipynb",
+ "jsonutil.json_clean": "70c_callback.captum.ipynb",
+ "CaptumInterpretation": "70c_callback.captum.ipynb",
+ "CaptumInterpretation.insights": "70c_callback.captum.ipynb",
+ "CometCallback": "70d_callback.comet.ipynb",
+ "synth_dbunch": "97_test_utils.ipynb",
+ "RegModel": "97_test_utils.ipynb",
+ "synth_learner": "97_test_utils.ipynb",
+ "VerboseCallback": "97_test_utils.ipynb",
+ "get_env": "97_test_utils.ipynb",
+ "try_import": "97_test_utils.ipynb",
+ "nvidia_smi": "97_test_utils.ipynb",
+ "nvidia_mem": "97_test_utils.ipynb",
+ "show_install": "97_test_utils.ipynb",
+ "PYTORCH_URL": "99_pytorch_doc.ipynb",
+ "pytorch_doc_link": "99_pytorch_doc.ipynb"}
+
+modules = ["torch_core.py",
+ "layers.py",
+ "losses.py",
+ "data/load.py",
+ "data/core.py",
+ "data/external.py",
+ "data/transforms.py",
+ "data/block.py",
+ "vision/core.py",
+ "vision/data.py",
+ "vision/augment.py",
+ "vision/utils.py",
+ "vision/widgets.py",
+ "vision/models/xresnet.py",
+ "optimizer.py",
+ "callback/core.py",
+ "learner.py",
+ "metrics.py",
+ "callback/schedule.py",
+ "callback/data.py",
+ "callback/hook.py",
+ "vision/models/unet.py",
+ "callback/progress.py",
+ "callback/tracker.py",
+ "callback/fp16.py",
+ "callback/training.py",
+ "callback/preds.py",
+ "callback/mixup.py",
+ "interpret.py",
+ "distributed.py",
+ "vision/learner.py",
+ "vision/gan.py",
+ "text/core.py",
+ "text/data.py",
+ "text/models/awdlstm.py",
+ "text/models/core.py",
+ "callback/rnn.py",
+ "text/learner.py",
+ "tabular/core.py",
+ "tabular/data.py",
+ "tabular/model.py",
+ "tabular/learner.py",
+ "collab.py",
+ "medical/imaging.py",
+ "medical/text.py",
+ "callback/wandb.py",
+ "callback/tensorboard.py",
+ "callback/neptune.py",
+ "callback/captum.py",
+ "callback/comet.py",
+ "test_utils.py",
+ "_pytorch_doc.py"]
+
+doc_url = "https://docs.fast.ai/"
+
+git_url = "https://github.com/fastai/fastai/tree/master/"
+
+def custom_doc_links(name):
+ from nbdev.showdoc import try_external_doc_link
+ return try_external_doc_link(name, ['fastcore', 'nbdev'])
+

fastai/_pytorch_doc.py

@@ -0,0 +1,46 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/99_pytorch_doc.ipynb.
+
+# %% ../nbs/99_pytorch_doc.ipynb 5
+from __future__ import annotations
+from types import ModuleType
+
+# %% auto 0
+__all__ = ['PYTORCH_URL', 'pytorch_doc_link']
+
+# %% ../nbs/99_pytorch_doc.ipynb 6
+PYTORCH_URL = 'https://pytorch.org/docs/stable/'
+
+# %% ../nbs/99_pytorch_doc.ipynb 7
+def _mod2page(
+ mod:ModuleType, # A PyTorch module
+) -> str:
+ "Get the webpage name for a PyTorch module"
+ if mod == Tensor: return 'tensors.html'
+ name = mod.__name__
+ name = name.replace('torch.', '').replace('utils.', '')
+ if name.startswith('nn.modules'): return 'nn.html'
+ return f'{name}.html'
+
+# %% ../nbs/99_pytorch_doc.ipynb 9
+import importlib
+
+# %% ../nbs/99_pytorch_doc.ipynb 10
+def pytorch_doc_link(
+ name:str # Name of a PyTorch module, class or function
+) -> (str, None):
+ "Get the URL to the documentation of a PyTorch module, class or function"
+ if name.startswith('F'): name = 'torch.nn.functional' + name[1:]
+ if not name.startswith('torch.'): name = 'torch.' + name
+ if name == 'torch.Tensor': return f'{PYTORCH_URL}tensors.html'
+ try:
+ mod = importlib.import_module(name)
+ return f'{PYTORCH_URL}{_mod2page(mod)}'
+ except: pass
+ splits = name.split('.')
+ mod_name,fname = '.'.join(splits[:-1]),splits[-1]
+ if mod_name == 'torch.Tensor': return f'{PYTORCH_URL}tensors.html#{name}'
+ try:
+ mod = importlib.import_module(mod_name)
+ page = _mod2page(mod)
+ return f'{PYTORCH_URL}{page}#{name}'
+ except: return None

fastai/basics.py

@@ -0,0 +1,6 @@
+from .data.all import *
+from .optimizer import *
+from .callback.core import *
+from .learner import *
+from .metrics import *
+from .interpret import *

fastai/callback/__init__.py

@@ -0,0 +1 @@
+

fastai/callback/all.py

@@ -0,0 +1,12 @@
+from .core import *
+from .data import *
+from .fp16 import *
+from .hook import *
+from .mixup import *
+from .progress import *
+from .schedule import *
+from .tracker import *
+from .rnn import *
+from .training import *
+from .preds import *
+from .channelslast import *

fastai/callback/azureml.py

@@ -0,0 +1,72 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/74_callback.azureml.ipynb (unless otherwise specified).
+
+__all__ = ['AzureMLCallback']
+
+# Cell
+from ..basics import *
+from ..learner import Callback
+
+# Cell
+from azureml.core.run import Run
+from azureml.exceptions import RunEnvironmentException
+import warnings
+
+# Cell
+class AzureMLCallback(Callback):
+ """
+ Log losses, metrics, model architecture summary to AzureML.
+
+ If `log_offline` is False, will only log if actually running on AzureML.
+ A custom AzureML `Run` class can be passed as `azurerun`.
+ If `log_to_parent` is True, will also log to the parent run, if exists (e.g. in AzureML pipelines).
+ """
+ order = Recorder.order+1
+
+ def __init__(self, azurerun=None, log_to_parent=True):
+ if azurerun:
+ self.azurerun = azurerun
+ else:
+ try:
+ self.azurerun = Run.get_context(allow_offline=False)
+ except RunEnvironmentException:
+ # running locally
+ self.azurerun = None
+ warnings.warn("Not running on AzureML and no azurerun passed, AzureMLCallback will be disabled.")
+ self.log_to_parent = log_to_parent
+
+ def before_fit(self):
+ self._log("n_epoch", self.learn.n_epoch)
+ self._log("model_class", str(type(self.learn.model)))
+
+ try:
+ summary_file = Path("outputs") / 'model_summary.txt'
+ with summary_file.open("w") as f:
+ f.write(repr(self.learn.model))
+ except:
+ print('Did not log model summary. Check if your model is PyTorch model.')
+
+ def after_batch(self):
+ # log loss and opt.hypers
+ if self.learn.training:
+ self._log('batch__loss', self.learn.loss.item())
+ self._log('batch__train_iter', self.learn.train_iter)
+ for i, h in enumerate(self.learn.opt.hypers):
+ for k, v in h.items():
+ self._log(f'batch__opt.hypers.{k}', v)
+
+ def after_epoch(self):
+ # log metrics
+ for n, v in zip(self.learn.recorder.metric_names, self.learn.recorder.log):
+ if n not in ['epoch', 'time']:
+ self._log(f'epoch__{n}', v)
+ if n == 'time':
+ # split elapsed time string, then convert into 'seconds' to log
+ m, s = str(v).split(':')
+ elapsed = int(m)*60 + int(s)
+ self._log(f'epoch__{n}', elapsed)
+
+ def _log(self, metric, value):
+ if self.azurerun is not None:
+ self.azurerun.log(metric, value)
+ if self.log_to_parent and self.azurerun.parent is not None:
+ self.azurerun.parent.log(metric, value)

fastai/callback/captum.py

@@ -0,0 +1,113 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/70c_callback.captum.ipynb.
+
+# %% ../../nbs/70c_callback.captum.ipynb 3
+from __future__ import annotations
+import tempfile
+from ..basics import *
+
+# %% auto 0
+__all__ = ['CaptumInterpretation']
+
+# %% ../../nbs/70c_callback.captum.ipynb 6
+from ipykernel import jsonutil
+
+# %% ../../nbs/70c_callback.captum.ipynb 7
+# Dirty hack as json_clean doesn't support CategoryMap type
+_json_clean=jsonutil.json_clean
+def json_clean(o):
+ o = list(o.items) if isinstance(o,CategoryMap) else o
+ return _json_clean(o)
+
+jsonutil.json_clean = json_clean
+
+# %% ../../nbs/70c_callback.captum.ipynb 8
+from captum.attr import IntegratedGradients,NoiseTunnel,GradientShap,Occlusion
+from captum.attr import visualization as viz
+
+from matplotlib.colors import LinearSegmentedColormap
+
+from captum.insights import AttributionVisualizer, Batch
+from captum.insights.attr_vis.features import ImageFeature
+
+# %% ../../nbs/70c_callback.captum.ipynb 16
+class CaptumInterpretation():
+ "Captum Interpretation for Resnet"
+ def __init__(self,learn,cmap_name='custom blue',colors=None,N=256,methods=('original_image','heat_map'),
+ signs=("all", "positive"),outlier_perc=1):
+ if colors is None: colors = [(0, '#ffffff'),(0.25, '#000000'),(1, '#000000')]
+ store_attr()
+ self.dls,self.model = learn.dls,self.learn.model
+ self.supported_metrics=['IG','NT','Occl']
+
+ def get_baseline_img(self, img_tensor,baseline_type):
+ baseline_img=None
+ if baseline_type=='zeros': baseline_img= img_tensor*0
+ if baseline_type=='uniform': baseline_img= torch.rand(img_tensor.shape)
+ if baseline_type=='gauss':
+ baseline_img= (torch.rand(img_tensor.shape).to(self.dls.device)+img_tensor)/2
+ return baseline_img.to(self.dls.device)
+
+ def visualize(self,inp,metric='IG',n_steps=1000,baseline_type='zeros',nt_type='smoothgrad', strides=(3,4,4), sliding_window_shapes=(3,15,15)):
+ if metric not in self.supported_metrics:
+ raise Exception(f"Metric {metric} is not supported. Currently {self.supported_metrics} are only supported")
+ tls = L([TfmdLists(inp, t) for t in L(ifnone(self.dls.tfms,[None]))])
+ inp_data=list(zip(*(tls[0],tls[1])))[0]
+ enc_data,dec_data=self._get_enc_dec_data(inp_data)
+ attributions=self._get_attributions(enc_data,metric,n_steps,nt_type,baseline_type,strides,sliding_window_shapes)
+ self._viz(attributions,dec_data,metric)
+
+ def _viz(self,attributions,dec_data,metric):
+ default_cmap = LinearSegmentedColormap.from_list(self.cmap_name,self.colors, N=self.N)
+ _ = viz.visualize_image_attr_multiple(np.transpose(attributions.squeeze().cpu().detach().numpy(), (1,2,0)),
+ np.transpose(dec_data[0].numpy(), (1,2,0)),
+ methods=self.methods,
+ cmap=default_cmap,
+ show_colorbar=True,
+ signs=self.signs,
+ outlier_perc=self.outlier_perc, titles=[f'Original Image - ({dec_data[1]})', metric])
+
+
+
+ def _get_enc_dec_data(self,inp_data):
+ dec_data=self.dls.after_item(inp_data)
+ enc_data=self.dls.after_batch(to_device(self.dls.before_batch(dec_data),self.dls.device))
+ return(enc_data,dec_data)
+
+ def _get_attributions(self,enc_data,metric,n_steps,nt_type,baseline_type,strides,sliding_window_shapes):
+ # Get Baseline
+ baseline=self.get_baseline_img(enc_data[0],baseline_type)
+ supported_metrics ={}
+ if metric == 'IG':
+ self._int_grads = self._int_grads if hasattr(self,'_int_grads') else IntegratedGradients(self.model)
+ return self._int_grads.attribute(enc_data[0],baseline, target=enc_data[1], n_steps=200)
+ elif metric == 'NT':
+ self._int_grads = self._int_grads if hasattr(self,'_int_grads') else IntegratedGradients(self.model)
+ self._noise_tunnel= self._noise_tunnel if hasattr(self,'_noise_tunnel') else NoiseTunnel(self._int_grads)
+ return self._noise_tunnel.attribute(enc_data[0].to(self.dls.device), n_samples=1, nt_type=nt_type, target=enc_data[1])
+ elif metric == 'Occl':
+ self._occlusion = self._occlusion if hasattr(self,'_occlusion') else Occlusion(self.model)
+ return self._occlusion.attribute(enc_data[0].to(self.dls.device),
+ strides = strides,
+ target=enc_data[1],
+ sliding_window_shapes=sliding_window_shapes,
+ baselines=baseline)
+
+# %% ../../nbs/70c_callback.captum.ipynb 26
+@patch
+def insights(x: CaptumInterpretation,inp_data,debug=True):
+ _baseline_func= lambda o: o*0
+ _get_vocab = lambda vocab: list(map(str,vocab)) if isinstance(vocab[0],bool) else vocab
+ dl = x.dls.test_dl(L(inp_data),with_labels=True, bs=4)
+ normalize_func= next((func for func in dl.after_batch if type(func)==Normalize),noop)
+
+ # captum v0.3 expects tensors without the batch dimension.
+ if nested_attr(normalize_func, 'mean.ndim', 4)==4: normalize_func.mean.squeeze_(0) 
+ if nested_attr(normalize_func, 'std.ndim', 4)==4: normalize_func.std.squeeze_(0) 
+
+ visualizer = AttributionVisualizer(
+ models=[x.model],
+ score_func=lambda o: torch.nn.functional.softmax(o, 1),
+ classes=_get_vocab(dl.vocab),
+ features=[ImageFeature("Image", baseline_transforms=[_baseline_func], input_transforms=[normalize_func])],
+ dataset=x._formatted_data_iter(dl,normalize_func))
+ visualizer.render(debug=debug)

fastai/callback/channelslast.py

@@ -0,0 +1,41 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/18c_callback.channelslast.ipynb.
+
+# %% ../../nbs/18c_callback.channelslast.ipynb 1
+from __future__ import annotations
+from ..basics import *
+from .fp16 import MixedPrecision
+
+from torch.cuda.amp import GradScaler
+
+# %% auto 0
+__all__ = ['ChannelsLast']
+
+# %% ../../nbs/18c_callback.channelslast.ipynb 7
+class ChannelsLast(Callback):
+ "Channels last training using PyTorch's Channels Last Memory Format (beta)"
+ order = -1 # Needs to run before any model modification callbacks occur
+ def before_fit(self):
+ self.learn.model.to(memory_format=torch.channels_last)
+
+# %% ../../nbs/18c_callback.channelslast.ipynb 9
+@patch
+@delegates(GradScaler)
+def to_channelslast(self:Learner,
+ to_fp16:bool=True, # Add `MixedPrecision` callback. Recommended for full channels last performance
+ **kwargs
+):
+ "Set `Learner` and inputs to `channels_last` format and `MixedPrecision` by default"
+ if to_fp16 and not hasattr(self, 'mixed_precision') and not hasattr(self, 'channels_last'):
+ return self.add_cbs([ChannelsLast(), MixedPrecision(**kwargs)])
+ elif not hasattr(self, 'channels_last'):
+ return self.add_cb(ChannelsLast())
+
+# %% ../../nbs/18c_callback.channelslast.ipynb 10
+@patch
+def to_contiguous(self:Learner, to_fp32:bool=False):
+ "Set `Learner` and inputs to `contiguous_format` (default format), optionally to single precision"
+ self.model.to(memory_format=torch.contiguous_format)
+ if to_fp32:
+ return self.remove_cbs([ChannelsLast, MixedPrecision])
+ else:
+ return self.remove_cb(ChannelsLast)

fastai/callback/comet.py

@@ -0,0 +1,91 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/70d_callback.comet.ipynb.
+
+# %% ../../nbs/70d_callback.comet.ipynb 3
+from __future__ import annotations
+
+import tempfile
+
+from ..basics import *
+from ..learner import Callback
+
+# %% auto 0
+__all__ = ['CometCallback']
+
+# %% ../../nbs/70d_callback.comet.ipynb 12
+import comet_ml
+
+# %% ../../nbs/70d_callback.comet.ipynb 13
+class CometCallback(Callback):
+ "Log losses, metrics, model weights, model architecture summary to neptune"
+ order = Recorder.order + 1
+
+ def __init__(self, project_name, log_model_weights=True):
+ self.log_model_weights = log_model_weights
+ self.keep_experiment_running = keep_experiment_running
+ self.project_name = project_name
+ self.experiment = None
+
+ def before_fit(self):
+ try:
+ self.experiment = comet_ml.Experiment(project_name=self.project_name)
+ except ValueError:
+ print("No active experiment")
+
+ try:
+ self.experiment.log_parameter("n_epoch", str(self.learn.n_epoch))
+ self.experiment.log_parameter("model_class", str(type(self.learn.model)))
+ except:
+ print(f"Did not log all properties.")
+
+ try:
+ with tempfile.NamedTemporaryFile(mode="w") as f:
+ with open(f.name, "w") as g:
+ g.write(repr(self.learn.model))
+ self.experiment.log_asset(f.name, "model_summary.txt")
+ except:
+ print("Did not log model summary. Check if your model is PyTorch model.")
+
+ if self.log_model_weights and not hasattr(self.learn, "save_model"):
+ print(
+ "Unable to log model to Comet.\n",
+ )
+
+ def after_batch(self):
+ # log loss and opt.hypers
+ if self.learn.training:
+ self.experiment.log_metric("batch__smooth_loss", self.learn.smooth_loss)
+ self.experiment.log_metric("batch__loss", self.learn.loss)
+ self.experiment.log_metric("batch__train_iter", self.learn.train_iter)
+ for i, h in enumerate(self.learn.opt.hypers):
+ for k, v in h.items():
+ self.experiment.log_metric(f"batch__opt.hypers.{k}", v)
+
+ def after_epoch(self):
+ # log metrics
+ for n, v in zip(self.learn.recorder.metric_names, self.learn.recorder.log):
+ if n not in ["epoch", "time"]:
+ self.experiment.log_metric(f"epoch__{n}", v)
+ if n == "time":
+ self.experiment.log_text(f"epoch__{n}", str(v))
+
+ # log model weights
+ if self.log_model_weights and hasattr(self.learn, "save_model"):
+ if self.learn.save_model.every_epoch:
+ _file = join_path_file(
+ f"{self.learn.save_model.fname}_{self.learn.save_model.epoch}",
+ self.learn.path / self.learn.model_dir,
+ ext=".pth",
+ )
+ else:
+ _file = join_path_file(
+ self.learn.save_model.fname,
+ self.learn.path / self.learn.model_dir,
+ ext=".pth",
+ )
+ self.experiment.log_asset(_file)
+
+ def after_fit(self):
+ try:
+ self.experiment.end()
+ except:
+ print("No neptune experiment to stop.")

fastai/callback/core.py

@@ -0,0 +1,187 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/13_callback.core.ipynb.
+
+# %% ../../nbs/13_callback.core.ipynb 2
+from __future__ import annotations
+from ..data.all import *
+from ..optimizer import *
+from ..losses import BaseLoss
+
+# %% auto 0
+__all__ = ['Callback', 'TrainEvalCallback', 'GatherPredsCallback', 'FetchPredsCallback', 'CancelStepException',
+ 'CancelBackwardException', 'CancelFitException', 'CancelEpochException', 'CancelTrainException',
+ 'CancelValidException', 'CancelBatchException', 'event']
+
+# %% ../../nbs/13_callback.core.ipynb 4
+_all_ = ['CancelStepException','CancelBackwardException','CancelFitException','CancelEpochException','CancelTrainException','CancelValidException','CancelBatchException']
+
+# %% ../../nbs/13_callback.core.ipynb 8
+_events = L.split('after_create before_fit before_epoch before_train before_batch after_pred after_loss \
+ before_backward after_cancel_backward after_backward before_step after_cancel_step after_step \
+ after_cancel_batch after_batch after_cancel_train after_train before_validate after_cancel_validate \
+ after_validate after_cancel_epoch after_epoch after_cancel_fit after_fit')
+
+mk_class('event', **_events.map_dict(),
+ doc="All possible events as attributes to get tab-completion and typo-proofing")
+
+# %% ../../nbs/13_callback.core.ipynb 9
+_all_ = ['event']
+
+# %% ../../nbs/13_callback.core.ipynb 14
+_inner_loop = "before_batch after_pred after_loss before_backward after_cancel_backward after_backward before_step after_step after_cancel_batch after_batch".split()
+
+# %% ../../nbs/13_callback.core.ipynb 15
+_ex_docs = dict(
+ CancelBatchException="Skip the rest of this batch and go to `after_batch`",
+ CancelTrainException="Skip the rest of the training part of the epoch and go to `after_train`",
+ CancelValidException="Skip the rest of the validation part of the epoch and go to `after_validate`",
+ CancelEpochException="Skip the rest of this epoch and go to `after_epoch`",
+ CancelStepException ="Skip stepping the optimizer",
+ CancelBackwardException="Skip the backward pass and go to `after_backward`",
+ CancelFitException ="Interrupts training and go to `after_fit`")
+
+for c,d in _ex_docs.items(): mk_class(c,sup=Exception,doc=d)
+
+# %% ../../nbs/13_callback.core.ipynb 16
+@funcs_kwargs(as_method=True)
+class Callback(Stateful,GetAttr):
+ "Basic class handling tweaks of the training loop by changing a `Learner` in various events"
+ order,_default,learn,run,run_train,run_valid = 0,'learn',None,True,True,True
+ _methods = _events
+
+ def __init__(self, **kwargs): assert not kwargs, f'Passed unknown events: {kwargs}'
+ def __repr__(self): return type(self).__name__
+
+ def __call__(self, event_name):
+ "Call `self.{event_name}` if it's defined"
+ _run = (event_name not in _inner_loop or (self.run_train and getattr(self, 'training', True)) or
+ (self.run_valid and not getattr(self, 'training', False)))
+ res = None
+ if self.run and _run: 
+ try: res = getcallable(self, event_name)()
+ except (CancelBatchException, CancelBackwardException, CancelEpochException, CancelFitException, CancelStepException, CancelTrainException, CancelValidException): raise
+ except Exception as e: raise modify_exception(e, f'Exception occured in `{self.__class__.__name__}` when calling event `{event_name}`:\n\t{e.args[0]}', replace=True)
+ if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
+ return res
+
+ def __setattr__(self, name, value):
+ "Set an attribute for a `Callback`"
+ if hasattr(self.learn,name):
+ warn(f"You are shadowing an attribute ({name}) that exists in the learner. Use `self.learn.{name}` to avoid this")
+ super().__setattr__(name, value)
+
+ @property
+ def name(self):
+ "Name of the `Callback`, camel-cased and with '*Callback*' removed"
+ return class2attr(self, 'Callback')
+
+# %% ../../nbs/13_callback.core.ipynb 34
+class TrainEvalCallback(Callback):
+ "`Callback` that tracks the number of iterations done and properly sets training/eval mode"
+ order,run_valid = -10,False
+ def after_create(self): self.learn.n_epoch = 1
+
+ def before_fit(self):
+ "Set the iter and epoch counters to 0, put the model and the right device"
+ self.learn.epoch,self.learn.loss = 0,tensor(0.)
+ self.learn.train_iter,self.learn.pct_train = 0,0.
+ device = getattr(self.dls, 'device', default_device())
+ self.model.to(device)
+ if isinstance(self.loss_func, (nn.Module, BaseLoss)): self.loss_func.to(device)
+ if hasattr(self.model, 'reset'): self.model.reset()
+
+ def after_batch(self):
+ "Update the iter counter (in training mode)"
+ self.learn.pct_train += 1./(self.n_iter*self.n_epoch)
+ self.learn.train_iter += 1
+
+ def before_train(self):
+ "Set the model to training mode"
+ self.learn.pct_train=self.epoch/self.n_epoch
+ self.model.train()
+ self.learn.training=True
+
+ def before_validate(self):
+ "Set the model to validation mode"
+ self.model.eval()
+ self.learn.training=False
+
+# %% ../../nbs/13_callback.core.ipynb 38
+if not hasattr(defaults, 'callbacks'): defaults.callbacks = [TrainEvalCallback]
+
+# %% ../../nbs/13_callback.core.ipynb 52
+class GatherPredsCallback(Callback):
+ "`Callback` that returns all predictions and targets, optionally `with_input` or `with_loss`"
+ _stateattrs=('preds','targets','inputs','losses')
+ def __init__(self,
+ with_input:bool=False, # Whether to return inputs
+ with_loss:bool=False, # Whether to return losses
+ save_preds:Path=None, # Path to save predictions
+ save_targs:Path=None, # Path to save targets
+ with_preds:bool=True, # Whether to return predictions
+ with_targs:bool=True, # Whether to return targets
+ concat_dim:int=0, # Dimension to concatenate returned tensors
+ pickle_protocol:int=2 # Pickle protocol used to save predictions and targets
+ ):
+ store_attr()
+
+ def before_batch(self):
+ "If `with_input`, detach batch inputs"
+ if self.with_input: self.inputs.append((self.learn.to_detach(self.xb)))
+
+ def before_validate(self):
+ "Initialize containers"
+ self.preds,self.targets = [],[]
+ if self.with_input: self.inputs = []
+ if self.with_loss: self.losses = []
+
+ def after_batch(self):
+ "Save predictions, targets and potentially losses"
+ if not hasattr(self, 'pred'): return
+ preds,targs = self.learn.to_detach(self.pred),self.learn.to_detach(self.yb)
+ if self.with_preds: self.preds.append(preds)
+ if self.with_targs: self.targets.append(targs)
+ if self.save_preds is not None: 
+ torch.save(preds, self.save_preds/str(self.iter), pickle_protocol=self.pickle_protocol)
+ if self.save_targs is not None: 
+ torch.save(targs[0], self.save_targs/str(self.iter), pickle_protocol=self.pickle_protocol)
+ if self.with_loss:
+ bs = find_bs(self.yb)
+ loss = self.loss if self.loss.numel() == bs else self.loss.view(bs,-1).mean(1)
+ self.losses.append(self.learn.to_detach(loss))
+
+ def after_validate(self):
+ "Concatenate all recorded tensors"
+ if not hasattr(self, 'preds'): return
+ if self.with_input: self.inputs = detuplify(to_concat(self.inputs, dim=self.concat_dim))
+ if self.with_preds: self.preds = detuplify(to_concat(self.preds, dim=self.concat_dim))
+ if self.with_targs: self.targets = detuplify(to_concat(self.targets, dim=self.concat_dim))
+ if self.with_loss: self.losses = to_concat(self.losses)
+
+ def all_tensors(self) -> (Tensor, list):
+ "Returns all recorded tensors in the order [inputs, preds, targets, losses]"
+ res = [self.preds if self.with_preds else None, self.targets if self.with_targs else None]
+ if self.with_input: res = [self.inputs] + res
+ if self.with_loss: res.append(self.losses)
+ return res
+
+# %% ../../nbs/13_callback.core.ipynb 54
+class FetchPredsCallback(Callback):
+ "A callback to fetch predictions during the training loop"
+ remove_on_fetch = True
+ def __init__(self,
+ ds_idx:int=1, # Index of dataset, 0 for train, 1 for valid, used if `dl` is not present
+ dl:DataLoader=None, # `DataLoader` used for fetching `Learner` predictions
+ with_input:bool=False, # Whether to return inputs in `GatherPredsCallback`
+ with_decoded:bool=False, # Whether to return decoded predictions
+ cbs:Callback|MutableSequence=None, # `Callback` to temporarily remove from `Learner`
+ reorder:bool=True # Whether to sort prediction results
+ ):
+ self.cbs = L(cbs)
+ store_attr('ds_idx,dl,with_input,with_decoded,reorder')
+
+ def after_validate(self):
+ "Fetch predictions from `Learner` without `self.cbs` and `remove_on_fetch` callbacks"
+ to_rm = L(cb for cb in self.learn.cbs if getattr(cb, 'remove_on_fetch', False))
+ with self.learn.removed_cbs(to_rm + self.cbs) as learn:
+ self.preds = learn.get_preds(ds_idx=self.ds_idx, dl=self.dl,
+ with_input=self.with_input, with_decoded=self.with_decoded, inner=True, reorder=self.reorder)

fastai/callback/data.py

@@ -0,0 +1,71 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/14a_callback.data.ipynb.
+
+# %% ../../nbs/14a_callback.data.ipynb 3
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['CollectDataCallback', 'WeightedDL', 'PartialDL']
+
+# %% ../../nbs/14a_callback.data.ipynb 5
+class CollectDataCallback(Callback):
+ "Collect all batches, along with `pred` and `loss`, into `self.data`. Mainly for testing"
+ def before_fit(self): self.data = L()
+ def after_batch(self): 
+ self.data.append(self.learn.to_detach((self.xb,self.yb,self.pred,self.loss)))
+
+# %% ../../nbs/14a_callback.data.ipynb 6
+@delegates()
+class WeightedDL(TfmdDL):
+ "Weighted dataloader where `wgts` is used for the training set only"
+ def __init__(self, dataset=None, bs=None, wgts=None, **kwargs):
+ wgts = array([1.]*len(dataset) if wgts is None else wgts)
+ self.wgts = wgts/wgts.sum()
+ super().__init__(dataset=dataset, bs=bs, **kwargs)
+
+ def get_idxs(self):
+ if self.n==0: return []
+ if not self.shuffle: return super().get_idxs()
+ return list(np.random.choice(self.n, self.n, p=self.wgts))
+
+# %% ../../nbs/14a_callback.data.ipynb 7
+@patch
+@delegates(Datasets.dataloaders)
+def weighted_dataloaders(self:Datasets, wgts, bs=64, **kwargs):
+ "Create a weighted dataloader `WeightedDL` with `wgts` for the training set"
+ xtra_kwargs = [{}] * (self.n_subsets-1)
+ return self.dataloaders(bs=bs, dl_type=WeightedDL, dl_kwargs=({'wgts':wgts}, *xtra_kwargs), **kwargs)
+
+# %% ../../nbs/14a_callback.data.ipynb 12
+@patch
+@delegates(Datasets.weighted_dataloaders)
+def weighted_dataloaders(self:DataBlock, source, wgts, bs=64, verbose:bool=False, **kwargs):
+ "Create a weighted dataloader `WeightedDL` with `wgts` for the dataset"
+ dss = self.datasets(source, verbose=verbose)
+ if not hasattr(wgts, '__array__'): wgts = np.array(wgts)
+ trn_wgts = wgts[dss.splits[0]]
+ return dss.weighted_dataloaders(trn_wgts, bs=bs, after_batch=self.batch_tfms, after_item=self.item_tfms, **kwargs)
+
+# %% ../../nbs/14a_callback.data.ipynb 14
+@delegates()
+class PartialDL(TfmdDL):
+ "Select randomly partial quantity of data at each epoch"
+ def __init__(self, dataset=None, bs=None, partial_n=None, **kwargs):
+ super().__init__(dataset=dataset, bs=bs, **kwargs)
+ self.partial_n = min(partial_n, self.n) if partial_n else None
+
+ def get_idxs(self):
+ if self.partial_n is None: return super().get_idxs()
+ return list(np.random.choice(self.n, self.partial_n, replace=False))
+
+ def __len__(self):
+ if self.partial_n is None: return super().__len__()
+ return self.partial_n//self.bs + (0 if self.drop_last or self.partial_n%self.bs==0 else 1)
+
+# %% ../../nbs/14a_callback.data.ipynb 15
+@patch
+@delegates(Datasets.dataloaders)
+def partial_dataloaders(self:FilteredBase, partial_n, bs=64, **kwargs):
+ "Create a partial dataloader `PartialDL` for the training set"
+ xtra_kwargs = [{}] * (self.n_subsets-1)
+ return self.dataloaders(bs=bs, dl_type=PartialDL, dl_kwargs=({'partial_n':partial_n}, *xtra_kwargs), **kwargs)

fastai/callback/fp16.py

@@ -0,0 +1,217 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/18_callback.fp16.ipynb.
+
+# %% ../../nbs/18_callback.fp16.ipynb 2
+from __future__ import annotations
+from ..basics import *
+from .progress import *
+
+from torch.cuda.amp import GradScaler,autocast
+from torch.cuda.amp.grad_scaler import OptState
+
+# %% auto 0
+__all__ = ['MixedPrecision', 'FP16TestCallback', 'get_master', 'to_master_grads', 'to_model_params', 'test_overflow',
+ 'grad_overflow', 'copy_clone', 'ModelToHalf', 'NonNativeMixedPrecision']
+
+# %% ../../nbs/18_callback.fp16.ipynb 17
+@delegates(GradScaler)
+class MixedPrecision(Callback):
+ "Mixed precision training using Pytorch's `autocast` and `GradScaler`"
+ order = 10
+ def __init__(self, **kwargs): self.kwargs = kwargs
+ def before_fit(self): 
+ self.autocast,self.learn.scaler,self.scales = autocast(),GradScaler(**self.kwargs),L()
+ def before_batch(self): self.autocast.__enter__()
+ def after_pred(self):
+ if next(flatten(self.pred)).dtype==torch.float16: self.learn.pred = to_float(self.pred)
+ def after_loss(self): self.autocast.__exit__(None, None, None)
+ def before_backward(self): self.learn.loss_grad = self.scaler.scale(self.loss_grad)
+ def before_step(self):
+ "Use `self` as a fake optimizer. `self.skipped` will be set to True `after_step` if gradients overflow. "
+ self.skipped=True
+ self.scaler.step(self)
+ if self.skipped: raise CancelStepException()
+ self.scales.append(self.scaler.get_scale())
+ def after_step(self): self.learn.scaler.update()
+
+ @property 
+ def param_groups(self): 
+ "Pretend to be an optimizer for `GradScaler`"
+ return self.opt.param_groups
+ def step(self, *args, **kwargs): 
+ "Fake optimizer step to detect whether this batch was skipped from `GradScaler`"
+ self.skipped=False
+ def after_fit(self): self.autocast,self.learn.scaler,self.scales = None,None,None
+
+# %% ../../nbs/18_callback.fp16.ipynb 19
+class FP16TestCallback(Callback):
+ "Asserts that predictions are `float16` values"
+ order = 9
+ def after_pred(self): assert listify(flatten(self.pred))[0].dtype==torch.float16
+
+# %% ../../nbs/18_callback.fp16.ipynb 22
+@patch
+@delegates(GradScaler)
+def to_fp16(self:Learner, **kwargs): return self.add_cb(MixedPrecision(**kwargs))
+
+# %% ../../nbs/18_callback.fp16.ipynb 23
+@patch
+def to_fp32(self:Learner): return self.remove_cb(MixedPrecision)
+
+# %% ../../nbs/18_callback.fp16.ipynb 26
+from ..fp16_utils import convert_network, model_grads_to_master_grads, master_params_to_model_params
+
+# %% ../../nbs/18_callback.fp16.ipynb 32
+from torch.nn.utils import parameters_to_vector
+
+# %% ../../nbs/18_callback.fp16.ipynb 33
+def get_master(
+ opt:Optimizer, # Optimizer from which to retrieve model params
+ flat_master:bool=False, # Flatten fp32 params into a vector for better performance
+) -> list: # List of fp16 params, and list of fp32 params
+ "Creates fp16 model params given an initialized `Optimizer`, also returning fp32 model params. "
+ model_params = [[param for param in pg if getattr(param, 'requires_grad', False) and hasattr(param, 'data')] for pg in opt.param_lists]
+ if flat_master:
+ master_params = []
+ for pg in model_params:
+ mp = parameters_to_vector([param.data.float() for param in pg])
+ mp = nn.Parameter(mp, requires_grad=True)
+ if mp.grad is None: mp.grad = mp.new(*mp.size())
+ master_params.append([mp])
+ else:
+ master_params = [[nn.Parameter(param.data.clone().float().detach(), requires_grad=True) for param in pg] for pg in model_params]
+ return model_params, master_params
+
+# %% ../../nbs/18_callback.fp16.ipynb 38
+def to_master_grads( 
+ model_pgs:list, # Fp16 model parameters to copy gradients from
+ master_pgs:list, # Fp32 model parameters to copy gradients to
+ flat_master:bool=False, # Whether or not fp32 parameters were previously flattened
+):
+ "Move fp16 model gradients to fp32 master gradients"
+ for (model_params,master_params) in zip(model_pgs,master_pgs):
+ model_grads_to_master_grads(model_params, master_params, flat_master=flat_master)
+
+# %% ../../nbs/18_callback.fp16.ipynb 42
+def to_model_params(
+ model_pgs:list, # Fp16 model params to copy to
+ master_pgs:list, # Fp32 master params to copy from
+ flat_master:bool=False # Whether master_pgs was previously flattened
+)->None:
+ "Copy updated fp32 master params to fp16 model params after gradient step. " 
+ for (model_params,master_params) in zip(model_pgs,master_pgs):
+ master_params_to_model_params(model_params, master_params, flat_master=flat_master)
+
+# %% ../../nbs/18_callback.fp16.ipynb 47
+def test_overflow(x:torch.Tensor):
+ "Tests whether fp16 gradients have overflown."
+ s = float(x.float().sum())
+ return (s == float('inf') or s == float('-inf') or s != s)
+
+# %% ../../nbs/18_callback.fp16.ipynb 50
+def grad_overflow(pgs:list)->bool: 
+ "Tests all fp16 parameters in pgs for gradient overflow"
+ for pg in pgs:
+ for p in pg:
+ if p.grad is not None and test_overflow(p.grad.data): return True
+ return False
+
+# %% ../../nbs/18_callback.fp16.ipynb 53
+def copy_clone(d):
+ return {k:(v.detach().clone().float() if isinstance(v,Tensor) else v) for k,v in d.items()}
+
+# %% ../../nbs/18_callback.fp16.ipynb 54
+def _copy_state(opt, pgs1, pgs2):
+ opt.param_lists = pgs2
+ for pg1,pg2 in zip(pgs1, pgs2):
+ for p1,p2 in zip(pg1, pg2): opt.state[p2] = copy_clone(opt.state.pop(p1, {}))
+
+# %% ../../nbs/18_callback.fp16.ipynb 55
+class ModelToHalf(Callback):
+ "Use with NonNativeMixedPrecision callback (but it needs to run at the very beginning)"
+ order=-50
+ def before_fit(self): self.learn.model = convert_network(self.model, dtype=torch.float16)
+ def after_fit (self): self.learn.model = convert_network(self.model, dtype=torch.float32)
+
+# %% ../../nbs/18_callback.fp16.ipynb 56
+@docs
+class NonNativeMixedPrecision(Callback):
+ "Run training in mixed precision"
+ order=10
+ def __init__(self, 
+ loss_scale:int=512, # Non-dynamic loss scale, used to avoid underflow of gradients. 
+ flat_master:bool=False, # Whether to flatten fp32 parameters for performance
+ dynamic:bool=True, # Whether to automatically determine loss scaling
+ max_loss_scale:float=2.**24, # Starting value for dynamic loss scaling
+ div_factor:float=2., # Divide by this on overflow, multiply by this after scale_wait batches
+ scale_wait:int=500, # Number of batches to wait for increasing loss scale
+ clip:float=None, # Value to clip gradients at, max_norm, as in `nn.utils.clip_grad_norm_`
+ ): 
+ assert torch.backends.cudnn.enabled, "Mixed precision training requires cudnn."
+ self.flat_master,self.dynamic,self.max_loss_scale = flat_master,dynamic,max_loss_scale
+ self.div_factor,self.scale_wait,self.clip = div_factor,scale_wait,clip
+ self.loss_scale = max_loss_scale if dynamic else loss_scale
+
+ def before_fit(self):
+ assert self.dls.device.type == 'cuda', "Mixed-precision training requires a GPU, remove the call `to_fp16`"
+ if self.learn.opt is None: self.learn.create_opt()
+ self.model_pgs,self.master_pgs = get_master(self.opt, self.flat_master)
+ self.old_pgs = self.opt.param_lists
+ #Changes the optimizer so that the optimization step is done in FP32.
+ _copy_state(self.learn.opt, self.model_pgs, self.master_pgs)
+ if self.dynamic: self.count = 0
+
+ def before_batch(self): self.learn.xb = to_half(self.xb)
+ def after_pred(self): self.learn.pred = to_float(self.pred)
+ def before_backward(self): self.learn.loss_grad *= self.loss_scale
+
+ def before_step(self):
+ #First, check for an overflow
+ if self.dynamic and grad_overflow(self.model_pgs):
+ self.loss_scale /= self.div_factor
+ self.learn.loss_grad /= self.div_factor #to record correct loss
+ self.model.zero_grad()
+ raise CancelBatchException() #skip step and zero_grad
+ to_master_grads(self.model_pgs, self.master_pgs, self.flat_master)
+ for master_params in self.master_pgs:
+ for param in master_params:
+ if param.grad is not None: param.grad.div_(self.loss_scale)
+ if self.clip is not None:
+ for group in self.master_pgs: nn.utils.clip_grad_norm_(group, self.clip)
+ # Check if it's been long enough without overflow
+ if self.dynamic:
+ self.count += 1
+ if self.count == self.scale_wait:
+ self.count = 0
+ self.loss_scale *= self.div_factor
+
+ def after_step(self):
+ self.model.zero_grad() #Zero the gradients of the model manually (optimizer disconnected)
+ to_model_params(self.model_pgs, self.master_pgs, self.flat_master)
+
+ def after_batch(self):
+ if self.training: self.learn.loss_grad /= self.loss_scale #Log correct loss
+ def after_fit(self):
+ if not hasattr(self,'master_pgs'): return
+ _copy_state(self.learn.opt, self.master_pgs, self.model_pgs)
+ self.learn.opt.param_lists = self.old_pgs
+ delattr(self, "master_pgs")
+ delattr(self, "model_pgs")
+ delattr(self, "old_pgs")
+
+ _docs = dict(before_fit="Put the model in FP16 and prepare the two copies of the parameters",
+ before_batch="Put the input in FP16",
+ after_pred="Put the output back to FP32 so that the loss is computed in FP32",
+ before_backward="Apply loss scaling to avoid gradient underflow",
+ before_step="Update and apply dynamic loss scaling, move gradients to fp32, apply gradient clipping",
+ after_step="Zero fp16 grads and update fp16 params with fp32 params. ",
+ after_batch="Ensure loss is logged correctly",
+ after_fit="Put the model back in FP32")
+
+# %% ../../nbs/18_callback.fp16.ipynb 60
+@patch
+@delegates(NonNativeMixedPrecision.__init__)
+def to_non_native_fp16(self:Learner, **kwargs): return self.add_cbs([ModelToHalf(), NonNativeMixedPrecision(**kwargs)])
+
+# %% ../../nbs/18_callback.fp16.ipynb 63
+@patch
+def to_non_native_fp32(self: Learner): return self.remove_cbs([ModelToHalf, NonNativeMixedPrecision])

fastai/callback/hook.py

@@ -0,0 +1,281 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/15_callback.hook.ipynb.
+
+# %% ../../nbs/15_callback.hook.ipynb 1
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['Hook', 'hook_output', 'Hooks', 'hook_outputs', 'dummy_eval', 'model_sizes', 'num_features_model', 'has_params',
+ 'HookCallback', 'total_params', 'layer_info', 'module_summary', 'ActivationStats']
+
+# %% ../../nbs/15_callback.hook.ipynb 13
+@docs
+class Hook():
+ "Create a hook on `m` with `hook_func`."
+ def __init__(self, m, hook_func, is_forward=True, detach=True, cpu=False, gather=False):
+ store_attr('hook_func,detach,cpu,gather')
+ f = m.register_forward_hook if is_forward else m.register_backward_hook
+ self.hook = f(self.hook_fn)
+ self.stored,self.removed = None,False
+
+ def hook_fn(self, module, input, output):
+ "Applies `hook_func` to `module`, `input`, `output`."
+ if self.detach:
+ input,output = to_detach(input, cpu=self.cpu, gather=self.gather),to_detach(output, cpu=self.cpu, gather=self.gather)
+ self.stored = self.hook_func(module, input, output)
+
+ def remove(self):
+ "Remove the hook from the model."
+ if not self.removed:
+ self.hook.remove()
+ self.removed=True
+
+ def __enter__(self, *args): return self
+ def __exit__(self, *args): self.remove()
+
+ _docs = dict(__enter__="Register the hook",
+ __exit__="Remove the hook")
+
+# %% ../../nbs/15_callback.hook.ipynb 25
+def _hook_inner(m,i,o): return o if isinstance(o,Tensor) or is_listy(o) else list(o)
+
+def hook_output(module, detach=True, cpu=False, grad=False):
+ "Return a `Hook` that stores activations of `module` in `self.stored`"
+ return Hook(module, _hook_inner, detach=detach, cpu=cpu, is_forward=not grad)
+
+# %% ../../nbs/15_callback.hook.ipynb 30
+@docs
+class Hooks():
+ "Create several hooks on the modules in `ms` with `hook_func`."
+ def __init__(self, ms, hook_func, is_forward=True, detach=True, cpu=False):
+ self.hooks = [Hook(m, hook_func, is_forward, detach, cpu) for m in ms]
+
+ def __getitem__(self,i): return self.hooks[i]
+ def __len__(self): return len(self.hooks)
+ def __iter__(self): return iter(self.hooks)
+ @property
+ def stored(self): return L(o.stored for o in self)
+
+ def remove(self):
+ "Remove the hooks from the model."
+ for h in self.hooks: h.remove()
+
+ def __enter__(self, *args): return self
+ def __exit__ (self, *args): self.remove()
+
+ _docs = dict(stored = "The states saved in each hook.",
+ __enter__="Register the hooks",
+ __exit__="Remove the hooks")
+
+# %% ../../nbs/15_callback.hook.ipynb 39
+def hook_outputs(modules, detach=True, cpu=False, grad=False):
+ "Return `Hooks` that store activations of all `modules` in `self.stored`"
+ return Hooks(modules, _hook_inner, detach=detach, cpu=cpu, is_forward=not grad)
+
+# %% ../../nbs/15_callback.hook.ipynb 43
+def dummy_eval(m, size=(64,64)):
+ "Evaluate `m` on a dummy input of a certain `size`"
+ ch_in = in_channels(m)
+ x = one_param(m).new(1, ch_in, *size).requires_grad_(False).uniform_(-1.,1.)
+ with torch.no_grad(): return m.eval()(x)
+
+# %% ../../nbs/15_callback.hook.ipynb 44
+def model_sizes(m, size=(64,64)):
+ "Pass a dummy input through the model `m` to get the various sizes of activations."
+ with hook_outputs(m) as hooks:
+ _ = dummy_eval(m, size=size)
+ return [o.stored.shape for o in hooks]
+
+# %% ../../nbs/15_callback.hook.ipynb 46
+def num_features_model(m):
+ "Return the number of output features for `m`."
+ sz,ch_in = 32,in_channels(m)
+ while True:
+ #Trying for a few sizes in case the model requires a big input size.
+ try:
+ return model_sizes(m, (sz,sz))[-1][1]
+ except Exception as e:
+ sz *= 2
+ if sz > 2048: raise e
+
+# %% ../../nbs/15_callback.hook.ipynb 50
+def has_params(m):
+ "Check if `m` has at least one parameter"
+ return len(list(m.parameters())) > 0
+
+# %% ../../nbs/15_callback.hook.ipynb 52
+@funcs_kwargs
+class HookCallback(Callback):
+ "`Callback` that can be used to register hooks on `modules`"
+ _methods = ["hook"]
+ hook = noops
+ def __init__(self, modules=None, every=None, remove_end=True, is_forward=True, detach=True, cpu=True, include_paramless=False , **kwargs):
+ store_attr('modules,every,remove_end,is_forward,detach,cpu, include_paramless')
+ assert not kwargs
+
+ def before_fit(self):
+ "Register the `Hooks` on `self.modules`."
+ if self.modules is None: self.modules = [m for m in flatten_model(self.model) if self.include_paramless or has_params(m)]
+ if self.every is None: self._register()
+
+ def before_batch(self):
+ if self.every is None: return
+ if self.training and self.train_iter%self.every==0: self._register()
+
+ def after_batch(self):
+ if self.every is None: return
+ if self.training and self.train_iter%self.every==0: self._remove()
+
+ def after_fit(self):
+ "Remove the `Hooks`."
+ if self.remove_end: self._remove()
+
+ def _register(self): self.hooks = Hooks(self.modules, self.hook, self.is_forward, self.detach, self.cpu)
+ def _remove(self):
+ if getattr(self, 'hooks', None): self.hooks.remove()
+
+ def __del__(self): self._remove()
+
+# %% ../../nbs/15_callback.hook.ipynb 59
+def total_params(m):
+ "Give the number of parameters of a module and if it's trainable or not"
+ params = sum([p.numel() for p in m.parameters()])
+ trains = [p.requires_grad for p in m.parameters()]
+ return params, (False if len(trains)==0 else trains[0])
+
+# %% ../../nbs/15_callback.hook.ipynb 61
+def layer_info(learn, *xb):
+ "Return layer infos of `model` on `xb` (only support batch first inputs)"
+ def _track(m, i, o): 
+ params, trainable, shape = '', '', ''
+ same = any((isinstance(x[0], torch.Tensor) and x[0].shape[1:] == x[1].shape for x in zip(i, o)))
+ shape = apply(lambda x: x.shape, o)
+ if hasattr(m, 'weight'): # non activation layer
+ params, trainable = total_params(m)
+ return (type(m).__name__, params, trainable, shape, same)
+ 
+ with Hooks(flatten_model(learn.model), _track) as h:
+ batch = apply(lambda o:o[:1], xb)
+ train_only_cbs = [cb for cb in learn.cbs if hasattr(cb, '_only_train_loop')]
+ with learn.removed_cbs(train_only_cbs), learn.no_logging(), learn as l:
+ r = l.get_preds(dl=[batch], inner=True, reorder=False)
+ return h.stored
+
+# %% ../../nbs/15_callback.hook.ipynb 66
+def _get_shapes(o, bs): 
+ inp = o[first(o)] if (isinstance(o, dict)) else o
+ return ' x '.join([str(bs)] + [str(t) for t in inp[1:]])
+
+def _print_shapes(o, bs):
+ if isinstance(o, torch.Size): return _get_shapes(o, bs)
+ elif isinstance(o, tuple): return _get_shapes(o[0], bs)
+ else: return str([_print_shapes(x, bs) for x in o])
+
+# %% ../../nbs/15_callback.hook.ipynb 67
+def module_summary(learn, *xb):
+ "Print a summary of `model` using `xb`"
+ #Individual parameters wrapped in ParameterModule aren't called through the hooks in `layer_info`,
+ # thus are not counted inside the summary
+ #TODO: find a way to have them counted in param number somehow
+ infos = layer_info(learn, *xb)
+ n,bs = 76,find_bs(xb)
+ inp_sz = _print_shapes(apply(lambda x:x.shape, xb), bs)
+ res = f"{type(learn.model).__name__} (Input shape: {inp_sz})\n"
+ res += "=" * n + "\n"
+ res += f"{'Layer (type)':<20} {'Output Shape':<20} {'Param #':<10} {'Trainable':<10}\n"
+ res += "=" * n
+ ps,trn_ps,j = 0,0,0
+ infos = [o for o in infos if o is not None] #see comment in previous cell
+ prev_sz = None
+ for typ,np,trn,sz,chnged in infos:
+ if sz is None: continue
+ if j == 0:
+ res += f'\n{"":<20} {_print_shapes(sz, bs)[:19]:<20}' # to avoid a double line at the top
+ if not chnged and not prev_sz == sz and j > 0: res += "\n" + "_" * n + "\n" + f'{"":<20} {_print_shapes(sz, bs)[:19]:<20}'
+ j = 1
+ res += f"\n{typ:<20} {'':<20} {np:<10} {str(trn):<10}"
+ if np != '':
+ ps += np
+ if trn: trn_ps += np
+ prev_sz = sz
+ res += "\n" + "_" * n + "\n"
+ res += f"\nTotal params: {ps:,}\n"
+ res += f"Total trainable params: {trn_ps:,}\n"
+ res += f"Total non-trainable params: {ps - trn_ps:,}\n\n"
+ return PrettyString(res)
+
+# %% ../../nbs/15_callback.hook.ipynb 68
+@patch
+def summary(self:Learner):
+ "Print a summary of the model, optimizer and loss function."
+ xb = self.dls.train.one_batch()[:getattr(self.dls.train, "n_inp", 1)]
+ res = module_summary(self, *xb)
+ res += f"Optimizer used: {self.opt_func}\nLoss function: {self.loss_func}\n\n"
+ if self.opt is not None:
+ res += f"Model " + ("unfrozen\n\n" if self.opt.frozen_idx==0 else f"frozen up to parameter group #{self.opt.frozen_idx}\n\n")
+ res += "Callbacks:\n" + '\n'.join(f" - {cb}" for cb in self.cbs.sorted('order'))
+ return PrettyString(res)
+
+# %% ../../nbs/15_callback.hook.ipynb 74
+@delegates()
+class ActivationStats(HookCallback):
+ "Callback that record the mean and std of activations."
+ order=-20
+ def __init__(self, with_hist=False, **kwargs):
+ super().__init__(**kwargs)
+ self.with_hist = with_hist
+
+ def before_fit(self):
+ "Initialize stats."
+ super().before_fit()
+ self.stats = L()
+
+ def hook(self, m, i, o):
+ if isinstance(o, tuple): return self.hook_multi_ouput(o)
+ o = o.float()
+ res = {'mean': o.mean().item(), 'std': o.std().item(),
+ 'near_zero': (o<=0.05).long().sum().item()/o.numel()}
+ if self.with_hist: res['hist'] = o.histc(40,0,10)
+ return res
+ 
+ def hook_multi_ouput(self,o_tuple):
+ "For outputs of RNN which are [nested] tuples of tensors"
+ res = []
+ for o in self._flatten_tuple(o_tuple):
+ if not(isinstance(o, Tensor)): continue
+ res.append(self.hook(None, None, o))
+ return res
+
+ def _flatten_tuple(self, o_tuple):
+ "Recursively flatten a [nested] tuple"
+ res = []
+ for it in o_tuple:
+ if isinstance(it, tuple): res += self._flatten_tuple(it)
+ else: res += [it]
+ return tuple(res)
+
+ def after_batch(self):
+ "Take the stored results and puts it in `self.stats`"
+ if self.training and (self.every is None or self.train_iter%self.every == 0): self.stats.append(self.hooks.stored)
+ super().after_batch()
+
+ def layer_stats(self, idx):
+ lstats = self.stats.itemgot(idx)
+ return L(lstats.itemgot(o) for o in ('mean','std','near_zero'))
+
+ def hist(self, idx):
+ res = self.stats.itemgot(idx).itemgot('hist')
+ return torch.stack(tuple(res)).t().float().log1p()
+
+ def color_dim(self, idx, figsize=(10,5), ax=None):
+ "The 'colorful dimension' plot"
+ res = self.hist(idx)
+ if ax is None: ax = subplots(figsize=figsize)[1][0]
+ ax.imshow(res, origin='lower')
+ ax.axis('off')
+
+ def plot_layer_stats(self, idx):
+ _,axs = subplots(1, 3, figsize=(12,3))
+ for o,ax,title in zip(self.layer_stats(idx),axs,('mean','std','% near zero')):
+ ax.plot(o)
+ ax.set_title(title)

fastai/callback/mixup.py

@@ -0,0 +1,111 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/19_callback.mixup.ipynb.
+
+# %% ../../nbs/19_callback.mixup.ipynb 2
+from __future__ import annotations
+from ..basics import *
+from torch.distributions.beta import Beta
+
+# %% auto 0
+__all__ = ['reduce_loss', 'MixHandler', 'MixUp', 'CutMix']
+
+# %% ../../nbs/19_callback.mixup.ipynb 6
+def reduce_loss(
+ loss:Tensor, 
+ reduction:str='mean' # PyTorch loss reduction
+)->Tensor:
+ "Reduce the loss based on `reduction`"
+ return loss.mean() if reduction == 'mean' else loss.sum() if reduction == 'sum' else loss
+
+# %% ../../nbs/19_callback.mixup.ipynb 7
+class MixHandler(Callback):
+ "A handler class for implementing `MixUp` style scheduling"
+ run_valid = False
+ def __init__(self, 
+ alpha:float=0.5 # Determine `Beta` distribution in range (0.,inf]
+ ):
+ self.distrib = Beta(tensor(alpha), tensor(alpha))
+
+ def before_train(self):
+ "Determine whether to stack y"
+ self.stack_y = getattr(self.learn.loss_func, 'y_int', False)
+ if self.stack_y: self.old_lf,self.learn.loss_func = self.learn.loss_func,self.lf
+
+ def after_train(self):
+ "Set the loss function back to the previous loss"
+ if self.stack_y: self.learn.loss_func = self.old_lf
+
+ def after_cancel_train(self):
+ "If training is canceled, still set the loss function back"
+ self.after_train()
+
+ def after_cancel_fit(self):
+ "If fit is canceled, still set the loss function back"
+ self.after_train()
+
+ def lf(self, pred, *yb):
+ "lf is a loss function that applies the original loss function on both outputs based on `self.lam`"
+ if not self.training: return self.old_lf(pred, *yb)
+ with NoneReduce(self.old_lf) as lf:
+ loss = torch.lerp(lf(pred,*self.yb1), lf(pred,*yb), self.lam)
+ return reduce_loss(loss, getattr(self.old_lf, 'reduction', 'mean'))
+
+# %% ../../nbs/19_callback.mixup.ipynb 10
+class MixUp(MixHandler):
+ "Implementation of https://arxiv.org/abs/1710.09412"
+ def __init__(self, 
+ alpha:float=.4 # Determine `Beta` distribution in range (0.,inf]
+ ): 
+ super().__init__(alpha)
+ 
+ def before_batch(self):
+ "Blend xb and yb with another random item in a second batch (xb1,yb1) with `lam` weights"
+ lam = self.distrib.sample((self.y.size(0),)).squeeze().to(self.x.device)
+ lam = torch.stack([lam, 1-lam], 1)
+ self.lam = lam.max(1)[0]
+ shuffle = torch.randperm(self.y.size(0)).to(self.x.device)
+ xb1,self.yb1 = tuple(L(self.xb).itemgot(shuffle)),tuple(L(self.yb).itemgot(shuffle))
+ nx_dims = len(self.x.size())
+ self.learn.xb = tuple(L(xb1,self.xb).map_zip(torch.lerp,weight=unsqueeze(self.lam, n=nx_dims-1)))
+
+ if not self.stack_y:
+ ny_dims = len(self.y.size())
+ self.learn.yb = tuple(L(self.yb1,self.yb).map_zip(torch.lerp,weight=unsqueeze(self.lam, n=ny_dims-1)))
+
+# %% ../../nbs/19_callback.mixup.ipynb 21
+class CutMix(MixHandler): 
+ "Implementation of https://arxiv.org/abs/1905.04899"
+ def __init__(self, 
+ alpha:float=1. # Determine `Beta` distribution in range (0.,inf]
+ ): 
+ super().__init__(alpha)
+ 
+ def before_batch(self):
+ "Add `rand_bbox` patches with size based on `lam` and location chosen randomly."
+ bs, _, H, W = self.x.size()
+ self.lam = self.distrib.sample((1,)).to(self.x.device)
+ shuffle = torch.randperm(bs).to(self.x.device)
+ xb1,self.yb1 = self.x[shuffle], tuple((self.y[shuffle],))
+ x1, y1, x2, y2 = self.rand_bbox(W, H, self.lam)
+ self.learn.xb[0][..., y1:y2, x1:x2] = xb1[..., y1:y2, x1:x2]
+ self.lam = (1 - ((x2-x1)*(y2-y1))/float(W*H))
+ if not self.stack_y:
+ ny_dims = len(self.y.size())
+ self.learn.yb = tuple(L(self.yb1,self.yb).map_zip(torch.lerp,weight=unsqueeze(self.lam, n=ny_dims-1)))
+
+ def rand_bbox(self, 
+ W:int, # Width bbox will be
+ H:int, # Height bbox will be
+ lam:Tensor # lambda sample from Beta distribution i.e tensor([0.3647])
+ )->tuple: # Represents the top-left pixel location and the bottom-right pixel location
+ "Give a bounding box location based on the size of the im and a weight"
+ cut_rat = torch.sqrt(1. - lam).to(self.x.device)
+ cut_w = torch.round(W * cut_rat).type(torch.long).to(self.x.device)
+ cut_h = torch.round(H * cut_rat).type(torch.long).to(self.x.device)
+ # uniform
+ cx = torch.randint(0, W, (1,)).to(self.x.device)
+ cy = torch.randint(0, H, (1,)).to(self.x.device)
+ x1 = torch.clamp(cx - cut_w // 2, 0, W)
+ y1 = torch.clamp(cy - cut_h // 2, 0, H)
+ x2 = torch.clamp(cx + cut_w // 2, 0, W)
+ y2 = torch.clamp(cy + cut_h // 2, 0, H)
+ return x1, y1, x2, y2

fastai/callback/neptune.py

@@ -0,0 +1,80 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/70b_callback.neptune.ipynb.
+
+# %% ../../nbs/70b_callback.neptune.ipynb 2
+from __future__ import annotations
+import tempfile
+from ..basics import *
+from ..learner import Callback
+
+# %% auto 0
+__all__ = ['NeptuneCallback']
+
+# %% ../../nbs/70b_callback.neptune.ipynb 12
+import neptune
+
+# %% ../../nbs/70b_callback.neptune.ipynb 13
+class NeptuneCallback(Callback):
+ "Log losses, metrics, model weights, model architecture summary to neptune"
+ order = Recorder.order+1
+ def __init__(self, log_model_weights=True, keep_experiment_running=False):
+ self.log_model_weights = log_model_weights
+ self.keep_experiment_running = keep_experiment_running
+ self.experiment = None
+
+ if neptune.project is None:
+ raise ValueError('You did not initialize project in neptune.\n',
+ 'Please invoke `neptune.init("USERNAME/PROJECT_NAME")` before this callback.')
+
+ def before_fit(self):
+ try:
+ self.experiment = neptune.get_experiment()
+ except ValueError:
+ print('No active experiment. Please invoke `neptune.create_experiment()` before this callback.')
+
+ try:
+ self.experiment.set_property('n_epoch', str(self.learn.n_epoch))
+ self.experiment.set_property('model_class', str(type(self.learn.model)))
+ except: print(f'Did not log all properties. Check properties in the {neptune.get_experiment()}.')
+
+ try:
+ with tempfile.NamedTemporaryFile(mode='w') as f:
+ with open(f.name, 'w') as g: g.write(repr(self.learn.model))
+ self.experiment.log_artifact(f.name, 'model_summary.txt')
+ except: print('Did not log model summary. Check if your model is PyTorch model.')
+
+ if self.log_model_weights and not hasattr(self.learn, 'save_model'):
+ print('Unable to log model to Neptune.\n',
+ 'Use "SaveModelCallback" to save model checkpoints that will be logged to Neptune.')
+
+ def after_batch(self):
+ # log loss and opt.hypers
+ if self.learn.training:
+ self.experiment.log_metric('batch__smooth_loss', self.learn.smooth_loss)
+ self.experiment.log_metric('batch__loss', self.learn.loss)
+ self.experiment.log_metric('batch__train_iter', self.learn.train_iter)
+ for i, h in enumerate(self.learn.opt.hypers):
+ for k, v in h.items(): self.experiment.log_metric(f'batch__opt.hypers.{k}', v)
+
+ def after_epoch(self):
+ # log metrics
+ for n, v in zip(self.learn.recorder.metric_names, self.learn.recorder.log):
+ if n not in ['epoch', 'time']: self.experiment.log_metric(f'epoch__{n}', v)
+ if n == 'time': self.experiment.log_text(f'epoch__{n}', str(v))
+
+ # log model weights
+ if self.log_model_weights and hasattr(self.learn, 'save_model'):
+ if self.learn.save_model.every_epoch:
+ _file = join_path_file(f'{self.learn.save_model.fname}_{self.learn.save_model.epoch}',
+ self.learn.path / self.learn.model_dir, ext='.pth')
+ else:
+ _file = join_path_file(self.learn.save_model.fname,
+ self.learn.path / self.learn.model_dir, ext='.pth')
+ self.experiment.log_artifact(_file)
+
+ def after_fit(self):
+ if not self.keep_experiment_running:
+ try: self.experiment.stop()
+ except: print('No neptune experiment to stop.')
+ else:
+ print(f'Your experiment (id: {self.experiment.id}, name: {self.experiment.name}) is left in the running state.\n',
+ 'You can log more data to it, like this: `neptune.log_metric()`')

fastai/callback/preds.py

@@ -0,0 +1,18 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/18b_callback.preds.ipynb.
+
+# %% ../../nbs/18b_callback.preds.ipynb 2
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['MCDropoutCallback']
+
+# %% ../../nbs/18b_callback.preds.ipynb 6
+class MCDropoutCallback(Callback):
+ def before_validate(self):
+ for m in [m for m in flatten_model(self.model) if 'dropout' in m.__class__.__name__.lower()]:
+ m.train()
+ 
+ def after_validate(self):
+ for m in [m for m in flatten_model(self.model) if 'dropout' in m.__class__.__name__.lower()]:
+ m.eval()

fastai/callback/progress.py

@@ -0,0 +1,124 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/16_callback.progress.ipynb.
+
+# %% ../../nbs/16_callback.progress.ipynb 1
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['ProgressCallback', 'ShowGraphCallback', 'CSVLogger']
+
+# %% ../../nbs/16_callback.progress.ipynb 7
+@docs
+class ProgressCallback(Callback):
+ "A `Callback` to handle the display of progress bars"
+ order,_stateattrs = 60,('mbar','pbar')
+
+ def before_fit(self):
+ assert hasattr(self.learn, 'recorder')
+ if self.create_mbar: self.mbar = master_bar(list(range(self.n_epoch)))
+ if self.learn.logger != noop:
+ self.old_logger,self.learn.logger = self.logger,self._write_stats
+ self._write_stats(self.recorder.metric_names)
+ else: self.old_logger = noop
+
+ def before_epoch(self):
+ if getattr(self, 'mbar', False): self.mbar.update(self.epoch)
+
+ def before_train(self): self._launch_pbar()
+ def before_validate(self): self._launch_pbar()
+ def after_train(self): self.pbar.on_iter_end()
+ def after_validate(self): self.pbar.on_iter_end()
+ def after_batch(self):
+ self.pbar.update(self.iter+1)
+ if hasattr(self, 'smooth_loss'): self.pbar.comment = f'{self.smooth_loss.item():.4f}'
+
+ def _launch_pbar(self):
+ self.pbar = progress_bar(self.dl, parent=getattr(self, 'mbar', None), leave=False)
+ self.pbar.update(0)
+
+ def after_fit(self):
+ if getattr(self, 'mbar', False):
+ self.mbar.on_iter_end()
+ delattr(self, 'mbar')
+ if hasattr(self, 'old_logger'): self.learn.logger = self.old_logger
+
+ def _write_stats(self, log):
+ if getattr(self, 'mbar', False): self.mbar.write([f'{l:.6f}' if isinstance(l, float) else str(l) for l in log], table=True)
+
+ _docs = dict(before_fit="Setup the master bar over the epochs",
+ before_epoch="Update the master bar",
+ before_train="Launch a progress bar over the training dataloader",
+ before_validate="Launch a progress bar over the validation dataloader",
+ after_train="Close the progress bar over the training dataloader",
+ after_validate="Close the progress bar over the validation dataloader",
+ after_batch="Update the current progress bar",
+ after_fit="Close the master bar")
+
+if not hasattr(defaults, 'callbacks'): defaults.callbacks = [TrainEvalCallback, Recorder, ProgressCallback]
+elif ProgressCallback not in defaults.callbacks: defaults.callbacks.append(ProgressCallback)
+
+# %% ../../nbs/16_callback.progress.ipynb 9
+@patch
+@contextmanager
+def no_bar(self:Learner):
+ "Context manager that deactivates the use of progress bars"
+ has_progress = hasattr(self, 'progress')
+ if has_progress: self.remove_cb(self.progress)
+ try: yield self
+ finally:
+ if has_progress: self.add_cb(ProgressCallback())
+
+# %% ../../nbs/16_callback.progress.ipynb 22
+class ShowGraphCallback(Callback):
+ "Update a graph of training and validation loss"
+ order,run_valid=65,False
+
+ def before_fit(self):
+ self.run = not hasattr(self.learn, 'lr_finder') and not hasattr(self, "gather_preds")
+ if not(self.run): return
+ self.nb_batches = []
+ assert hasattr(self.learn, 'progress')
+
+ def after_train(self): self.nb_batches.append(self.train_iter)
+
+ def after_epoch(self):
+ "Plot validation loss in the pbar graph"
+ if not self.nb_batches: return
+ rec = self.learn.recorder
+ iters = range_of(rec.losses)
+ val_losses = [v[1] for v in rec.values]
+ x_bounds = (0, (self.n_epoch - len(self.nb_batches)) * self.nb_batches[0] + len(rec.losses))
+ y_bounds = (0, max((max(Tensor(rec.losses)), max(Tensor(val_losses)))))
+ self.progress.mbar.update_graph([(iters, rec.losses), (self.nb_batches, val_losses)], x_bounds, y_bounds)
+
+# %% ../../nbs/16_callback.progress.ipynb 26
+class CSVLogger(Callback):
+ "Log the results displayed in `learn.path/fname`"
+ order=60
+ def __init__(self, fname='history.csv', append=False):
+ self.fname,self.append = Path(fname),append
+
+ def read_log(self):
+ "Convenience method to quickly access the log."
+ return pd.read_csv(self.path/self.fname)
+
+ def before_fit(self):
+ "Prepare file with metric names."
+ if hasattr(self, "gather_preds"): return
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ self.file = (self.path/self.fname).open('a' if self.append else 'w')
+ self.file.write(','.join(self.recorder.metric_names) + '\n')
+ self.old_logger,self.learn.logger = self.logger,self._write_line
+
+ def _write_line(self, log):
+ "Write a line with `log` and call the old logger."
+ self.file.write(','.join([str(t) for t in log]) + '\n')
+ self.file.flush()
+ os.fsync(self.file.fileno())
+ self.old_logger(log)
+
+ def after_fit(self):
+ "Close the file and clean up."
+ if hasattr(self, "gather_preds"): return
+ self.file.close()
+ self.learn.logger = self.old_logger

fastai/callback/rnn.py

@@ -0,0 +1,42 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/34_callback.rnn.ipynb.
+
+# %% ../../nbs/34_callback.rnn.ipynb 1
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['ModelResetter', 'RNNCallback', 'RNNRegularizer', 'rnn_cbs']
+
+# %% ../../nbs/34_callback.rnn.ipynb 5
+@docs
+class ModelResetter(Callback):
+ "`Callback` that resets the model at each validation/training step"
+ def before_train(self): self.model.reset()
+ def before_validate(self): self.model.reset()
+ def after_fit(self): self.model.reset()
+ _docs = dict(before_train="Reset the model before training",
+ before_validate="Reset the model before validation",
+ after_fit="Reset the model after fitting")
+
+# %% ../../nbs/34_callback.rnn.ipynb 6
+class RNNCallback(Callback):
+ "Save the raw and dropped-out outputs and only keep the true output for loss computation"
+ def after_pred(self): self.learn.pred,self.raw_out,self.out = [o[-1] if is_listy(o) else o for o in self.pred]
+
+# %% ../../nbs/34_callback.rnn.ipynb 7
+class RNNRegularizer(Callback):
+ "Add AR and TAR regularization"
+ order,run_valid = RNNCallback.order+1,False
+ def __init__(self, alpha=0., beta=0.): store_attr()
+ def after_loss(self):
+ if not self.training: return
+ if self.alpha: self.learn.loss_grad += self.alpha * self.rnn.out.float().pow(2).mean()
+ if self.beta:
+ h = self.rnn.raw_out
+ if len(h)>1: self.learn.loss_grad += self.beta * (h[:,1:] - h[:,:-1]).float().pow(2).mean()
+
+# %% ../../nbs/34_callback.rnn.ipynb 8
+def rnn_cbs(alpha=0., beta=0.):
+ "All callbacks needed for (optionally regularized) RNN training"
+ reg = [RNNRegularizer(alpha=alpha, beta=beta)] if alpha or beta else []
+ return [ModelResetter(), RNNCallback()] + reg

fastai/callback/schedule.py

@@ -0,0 +1,314 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/14_callback.schedule.ipynb.
+
+# %% ../../nbs/14_callback.schedule.ipynb 2
+from __future__ import annotations
+from ..basics import *
+from .tracker import SaveModelCallback
+
+# %% auto 0
+__all__ = ['annealer', 'sched_lin', 'sched_cos', 'sched_no', 'sched_exp', 'SchedLin', 'SchedCos', 'SchedNo', 'SchedExp',
+ 'SchedPoly', 'combine_scheds', 'combined_cos', 'ParamScheduler', 'LRFinder', 'valley', 'slide', 'minimum',
+ 'steep', 'SuggestionMethod']
+
+# %% ../../nbs/14_callback.schedule.ipynb 3
+_all_ = ['SuggestionMethod']
+
+# %% ../../nbs/14_callback.schedule.ipynb 8
+class _Annealer:
+ def __init__(self, f, start, end): store_attr('f,start,end')
+ def __call__(self, pos): return self.f(self.start, self.end, pos)
+
+# %% ../../nbs/14_callback.schedule.ipynb 9
+def annealer(f):
+ "Decorator to make `f` return itself partially applied."
+ @functools.wraps(f)
+ def _inner(start, end): return _Annealer(f, start, end)
+ return _inner
+
+# %% ../../nbs/14_callback.schedule.ipynb 11
+#TODO Jeremy, make this pickle
+#@annealer
+#def SchedLin(start, end, pos): return start + pos*(end-start)
+#@annealer
+#def SchedCos(start, end, pos): return start + (1 + math.cos(math.pi*(1-pos))) * (end-start) / 2
+#@annealer
+#def SchedNo (start, end, pos): return start
+#@annealer
+#def SchedExp(start, end, pos): return start * (end/start) ** pos
+#
+#SchedLin.__doc__ = "Linear schedule function from `start` to `end`"
+#SchedCos.__doc__ = "Cosine schedule function from `start` to `end`"
+#SchedNo .__doc__ = "Constant schedule function with `start` value"
+#SchedExp.__doc__ = "Exponential schedule function from `start` to `end`"
+
+# %% ../../nbs/14_callback.schedule.ipynb 12
+def sched_lin(start, end, pos): return start + pos*(end-start)
+def sched_cos(start, end, pos): return start + (1 + math.cos(math.pi*(1-pos))) * (end-start) / 2
+def sched_no (start, end, pos): return start
+def sched_exp(start, end, pos): return start * (end/start) ** pos
+
+def SchedLin(start, end): return _Annealer(sched_lin, start, end)
+def SchedCos(start, end): return _Annealer(sched_cos, start, end)
+def SchedNo (start, end): return _Annealer(sched_no, start, end)
+def SchedExp(start, end): return _Annealer(sched_exp, start, end)
+
+SchedLin.__doc__ = "Linear schedule function from `start` to `end`"
+SchedCos.__doc__ = "Cosine schedule function from `start` to `end`"
+SchedNo .__doc__ = "Constant schedule function with `start` value"
+SchedExp.__doc__ = "Exponential schedule function from `start` to `end`"
+
+# %% ../../nbs/14_callback.schedule.ipynb 15
+def SchedPoly(start, end, power):
+ "Polynomial schedule (of `power`) function from `start` to `end`"
+ def _inner(pos): return start + (end - start) * pos ** power
+ return _inner
+
+# %% ../../nbs/14_callback.schedule.ipynb 28
+def combine_scheds(pcts, scheds):
+ "Combine `scheds` according to `pcts` in one function"
+ assert sum(pcts) == 1.
+ pcts = tensor([0] + L(pcts))
+ assert torch.all(pcts >= 0)
+ pcts = torch.cumsum(pcts, 0)
+ pct_lim = len(pcts) - 2
+ def _inner(pos):
+ idx = min((pos >= pcts).nonzero().max(), pct_lim)
+ actual_pos = (pos-pcts[idx]) / (pcts[idx+1]-pcts[idx])
+ return scheds[idx](actual_pos.item())
+ return _inner
+
+# %% ../../nbs/14_callback.schedule.ipynb 33
+def combined_cos(pct, start, middle, end):
+ "Return a scheduler with cosine annealing from `start`→`middle` & `middle`→`end`"
+ return combine_scheds([pct,1-pct], [SchedCos(start, middle), SchedCos(middle, end)])
+
+# %% ../../nbs/14_callback.schedule.ipynb 38
+@docs
+class ParamScheduler(Callback):
+ "Schedule hyper-parameters according to `scheds`"
+ order,run_valid = 60,False
+
+ def __init__(self, scheds): self.scheds = scheds
+ def before_fit(self): self.hps = {p:[] for p in self.scheds.keys()}
+ def before_batch(self): self._update_val(self.pct_train)
+
+ def _update_val(self, pct):
+ for n,f in self.scheds.items(): self.opt.set_hyper(n, f(pct))
+
+ def after_batch(self):
+ for p in self.scheds.keys(): self.hps[p].append(self.opt.hypers[-1][p])
+
+ def after_fit(self):
+ if hasattr(self.learn, 'recorder') and hasattr(self, 'hps'): self.recorder.hps = self.hps
+
+ _docs = {"before_fit": "Initialize container for hyper-parameters",
+ "before_batch": "Set the proper hyper-parameters in the optimizer",
+ "after_batch": "Record hyper-parameters of this batch",
+ "after_fit": "Save the hyper-parameters in the recorder if there is one"}
+
+# %% ../../nbs/14_callback.schedule.ipynb 46
+@patch
+def fit_one_cycle(self:Learner, n_epoch, lr_max=None, div=25., div_final=1e5, pct_start=0.25, wd=None,
+ moms=None, cbs=None, reset_opt=False, start_epoch=0):
+ "Fit `self.model` for `n_epoch` using the 1cycle policy."
+ if self.opt is None: self.create_opt()
+ self.opt.set_hyper('lr', self.lr if lr_max is None else lr_max)
+ lr_max = np.array([h['lr'] for h in self.opt.hypers])
+ scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),
+ 'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}
+ self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=start_epoch)
+
+# %% ../../nbs/14_callback.schedule.ipynb 50
+@patch
+def plot_sched(self:Recorder, keys=None, figsize=None):
+ keys = self.hps.keys() if keys is None else L(keys)
+ rows,cols = (len(keys)+1)//2, min(2, len(keys))
+ figsize = figsize or (6*cols,4*rows)
+ _, axs = plt.subplots(rows, cols, figsize=figsize)
+ axs = axs.flatten() if len(keys) > 1 else L(axs)
+ for p,ax in zip(keys, axs):
+ ax.plot(self.hps[p])
+ ax.set_ylabel(p)
+
+# %% ../../nbs/14_callback.schedule.ipynb 54
+@patch
+def fit_flat_cos(self:Learner, n_epoch, lr=None, div_final=1e5, pct_start=0.75, wd=None,
+ cbs=None, reset_opt=False, start_epoch=0):
+ "Fit `self.model` for `n_epoch` at flat `lr` before a cosine annealing."
+ if self.opt is None: self.create_opt()
+ self.opt.set_hyper('lr', self.lr if lr is None else lr)
+ lr = np.array([h['lr'] for h in self.opt.hypers])
+ scheds = {'lr': combined_cos(pct_start, lr, lr, lr/div_final)}
+ self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=0)
+
+# %% ../../nbs/14_callback.schedule.ipynb 57
+@patch
+def fit_sgdr(self:Learner, n_cycles, cycle_len, lr_max=None, cycle_mult=2, cbs=None, reset_opt=False, wd=None,
+ start_epoch=0):
+ "Fit `self.model` for `n_cycles` of `cycle_len` using SGDR."
+ if self.opt is None: self.create_opt()
+ self.opt.set_hyper('lr', self.lr if lr_max is None else lr_max)
+ lr_max = np.array([h['lr'] for h in self.opt.hypers])
+ n_epoch = cycle_len * (cycle_mult**n_cycles-1)//(cycle_mult-1)
+ pcts = [cycle_len * cycle_mult**i / n_epoch for i in range(n_cycles)]
+ scheds = [SchedCos(lr_max, 0) for _ in range(n_cycles)]
+ scheds = {'lr': combine_scheds(pcts, scheds)}
+ self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=start_epoch)
+
+# %% ../../nbs/14_callback.schedule.ipynb 60
+@patch
+@delegates(Learner.fit_one_cycle)
+def fine_tune(self:Learner, epochs, base_lr=2e-3, freeze_epochs=1, lr_mult=100,
+ pct_start=0.3, div=5.0, **kwargs):
+ "Fine tune with `Learner.freeze` for `freeze_epochs`, then with `Learner.unfreeze` for `epochs`, using discriminative LR."
+ self.freeze()
+ self.fit_one_cycle(freeze_epochs, slice(base_lr), pct_start=0.99, **kwargs)
+ base_lr /= 2
+ self.unfreeze()
+ self.fit_one_cycle(epochs, slice(base_lr/lr_mult, base_lr), pct_start=pct_start, div=div, **kwargs)
+
+# %% ../../nbs/14_callback.schedule.ipynb 67
+@docs
+class LRFinder(ParamScheduler):
+ "Training with exponentially growing learning rate"
+ def __init__(self, start_lr=1e-7, end_lr=10, num_it=100, stop_div=True):
+ if num_it < 6: num_it = 6
+ self.scheds = {'lr': [SchedExp(s, e) for (s,e) in zip(start_lr,end_lr)
+ ] if is_listy(start_lr) else SchedExp(start_lr, end_lr)}
+ self.num_it,self.stop_div = num_it,stop_div
+
+ def before_fit(self):
+ super().before_fit()
+ path = self.path/self.model_dir
+ path.mkdir(parents=True, exist_ok=True)
+ self.tmp_d = tempfile.TemporaryDirectory(dir=path)
+ self.tmp_p = Path(self.tmp_d.name).stem
+ self.learn.save(f'{self.tmp_p}/_tmp')
+ self.best_loss = float('inf')
+
+ def before_batch(self): self._update_val(self.train_iter/self.num_it)
+
+ def after_batch(self):
+ super().after_batch()
+ if self.smooth_loss < self.best_loss: self.best_loss = self.smooth_loss
+ if self.smooth_loss > 4*self.best_loss and self.stop_div: raise CancelFitException()
+ if self.train_iter >= self.num_it: raise CancelFitException()
+
+ def before_validate(self): raise CancelValidException()
+
+ def after_fit(self):
+ self.learn.opt.zero_grad() # Needed before detaching the optimizer for future fits
+ tmp_f = self.path/self.model_dir/self.tmp_p/'_tmp.pth'
+ if tmp_f.exists():
+ self.learn.load(f'{self.tmp_p}/_tmp', with_opt=True)
+ self.tmp_d.cleanup()
+
+ _docs = {"before_fit": "Initialize container for hyper-parameters and save the model",
+ "before_batch": "Set the proper hyper-parameters in the optimizer",
+ "after_batch": "Record hyper-parameters of this batch and potentially stop training",
+ "after_fit": "Save the hyper-parameters in the recorder if there is one and load the original model",
+ "before_validate": "Skip the validation part of training"}
+
+# %% ../../nbs/14_callback.schedule.ipynb 78
+def valley(lrs:list, losses:list, num_it:int):
+ "Suggests a learning rate from the longest valley and returns its index"
+ n = len(losses)
+ max_start, max_end = 0,0
+
+ # find the longest valley
+ lds = [1]*n
+ for i in range(1,n):
+ for j in range(0,i):
+ if (losses[i] < losses[j]) and (lds[i] < lds[j] + 1):
+ lds[i] = lds[j] + 1
+ if lds[max_end] < lds[i]:
+ max_end = i
+ max_start = max_end - lds[max_end]
+ 
+ sections = (max_end - max_start) / 3
+ idx = max_start + int(sections) + int(sections/2)
+
+ return float(lrs[idx]), (float(lrs[idx]), losses[idx])
+
+# %% ../../nbs/14_callback.schedule.ipynb 81
+def slide(lrs:list, losses:list, num_it:int, lr_diff:int=15, thresh:float=.005, adjust_value:float=1.):
+ "Suggests a learning rate following an interval slide rule and returns its index"
+ losses = to_np(losses)
+ loss_grad = np.gradient(losses)
+
+ r_idx = -1
+ l_idx = r_idx - lr_diff
+ local_min_lr = lrs[l_idx]
+ while (l_idx >= -len(losses)) and (abs(loss_grad[r_idx] - loss_grad[l_idx]) > thresh):
+ local_min_lr = lrs[l_idx]
+ r_idx -= 1
+ l_idx -= 1
+ 
+ suggestion = float(local_min_lr) * adjust_value
+ idx = np.interp(np.log10(suggestion), np.log10(lrs), losses)
+ return suggestion, (suggestion, idx)
+
+# %% ../../nbs/14_callback.schedule.ipynb 84
+def minimum(lrs:list, losses:list, num_it:int):
+ "Suggests a learning rate one-tenth the minumum before divergance and returns its index"
+ lr_min = lrs[losses.argmin()].item()
+ loss_idx = losses[min(range(len(lrs)), key=lambda i: abs(lrs[i]-lr_min))]
+ return lr_min/10, (lr_min, loss_idx)
+
+# %% ../../nbs/14_callback.schedule.ipynb 86
+def steep(lrs:list, losses:list, num_it:int) -> (float, tuple):
+ "Suggests a learning rate when the slope is the steepest and returns its index"
+ grads = (losses[1:]-losses[:-1]) / (lrs[1:].log()-lrs[:-1].log())
+ lr_steep = lrs[grads.argmin()].item()
+ loss_idx = losses[min(range(len(lrs)), key=lambda i: abs(lrs[i]-lr_steep))]
+ return lr_steep, (lr_steep, loss_idx)
+
+# %% ../../nbs/14_callback.schedule.ipynb 88
+@patch
+def plot_lr_find(self:Recorder, skip_end=5, return_fig=True, suggestions=None, nms=None, **kwargs):
+ "Plot the result of an LR Finder test (won't work if you didn't do `learn.lr_find()` before)"
+ lrs = self.lrs if skip_end==0 else self.lrs [:-skip_end]
+ losses = self.losses if skip_end==0 else self.losses[:-skip_end]
+ fig, ax = plt.subplots(1,1)
+ ax.plot(lrs, losses)
+ ax.set_ylabel("Loss")
+ ax.set_xlabel("Learning Rate")
+ ax.set_xscale('log')
+ if suggestions:
+ colors = plt.rcParams['axes.prop_cycle'].by_key()['color'][1:]
+ for (val, idx), nm, color in zip(suggestions, nms, colors):
+ ax.plot(val, idx, 'o', label=nm, c=color)
+ ax.legend(loc='best')
+
+# %% ../../nbs/14_callback.schedule.ipynb 89
+mk_class("SuggestionMethod", **{o.__name__.capitalize():o for o in [valley,slide,minimum,steep]},
+ doc="All possible suggestion methods as convience attributes to get tab-completion and typo-proofing")
+
+# %% ../../nbs/14_callback.schedule.ipynb 90
+@patch
+def lr_find(self:Learner, start_lr=1e-7, end_lr=10, num_it=100, stop_div=True, show_plot=True, suggest_funcs=(SuggestionMethod.Valley)):
+ "Launch a mock training to find a good learning rate and return suggestions based on `suggest_funcs` as a named tuple"
+ n_epoch = num_it//len(self.dls.train) + 1
+ cb=LRFinder(start_lr=start_lr, end_lr=end_lr, num_it=num_it, stop_div=stop_div)
+ with self.no_logging(): self.fit(n_epoch, cbs=cb)
+ if suggest_funcs is not None:
+ lrs, losses = tensor(self.recorder.lrs[num_it//10:-5]), tensor(self.recorder.losses[num_it//10:-5])
+ nan_idxs = torch.nonzero(torch.isnan(losses.view(-1)))
+ if len(nan_idxs) > 0:
+ drop_idx = min(nan_idxs)
+ lrs = lrs[:drop_idx]
+ losses = losses[:drop_idx]
+ _suggestions, nms = [], []
+ for func in tuplify(suggest_funcs):
+ nms.append(func.__name__ if not isinstance(func, partial) else func.func.__name__) # deal with partials
+ _suggestions.append(func(lrs, losses, num_it))
+ 
+ SuggestedLRs = collections.namedtuple('SuggestedLRs', nms)
+ lrs, pnts = [], []
+ for lr, pnt in _suggestions:
+ lrs.append(lr)
+ pnts.append(pnt)
+ if show_plot: self.recorder.plot_lr_find(suggestions=pnts, nms=nms)
+ return SuggestedLRs(*lrs)
+
+ elif show_plot: self.recorder.plot_lr_find()

fastai/callback/tensorboard.py

@@ -0,0 +1,172 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/70a_callback.tensorboard.ipynb.
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 3
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['TensorBoardBaseCallback', 'TensorBoardCallback', 'TensorBoardProjectorCallback', 'projector_word_embeddings',
+ 'tensorboard_log']
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 18
+import tensorboard
+from torch.utils.tensorboard import SummaryWriter
+from .fp16 import ModelToHalf
+from .hook import hook_output
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 19
+class TensorBoardBaseCallback(Callback):
+ order = Recorder.order+1
+ "Base class for tensorboard callbacks"
+ def __init__(self): self.run_projector = False
+ 
+ def after_pred(self):
+ if self.run_projector: self.feat = _add_projector_features(self.learn, self.h, self.feat)
+ 
+ def after_validate(self):
+ if not self.run_projector: return
+ self.run_projector = False
+ self._remove()
+ _write_projector_embedding(self.learn, self.writer, self.feat)
+ 
+ def after_fit(self): 
+ if self.run: self.writer.close()
+ 
+ def _setup_projector(self):
+ self.run_projector = True
+ self.h = hook_output(self.learn.model[1][1] if not self.layer else self.layer)
+ self.feat = {}
+ 
+ def _setup_writer(self): self.writer = SummaryWriter(log_dir=self.log_dir)
+ def __del__(self): self._remove()
+ def _remove(self):
+ if getattr(self, 'h', None): self.h.remove()
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 21
+class TensorBoardCallback(TensorBoardBaseCallback):
+ "Saves model topology, losses & metrics for tensorboard and tensorboard projector during training"
+ def __init__(self, log_dir=None, trace_model=True, log_preds=True, n_preds=9, projector=False, layer=None):
+ super().__init__()
+ store_attr()
+
+ def before_fit(self):
+ self.run = not hasattr(self.learn, 'lr_finder') and not hasattr(self, "gather_preds") and rank_distrib()==0
+ if not self.run: return
+ self._setup_writer()
+ if self.trace_model:
+ if hasattr(self.learn, 'mixed_precision'):
+ raise Exception("Can't trace model in mixed precision, pass `trace_model=False` or don't use FP16.")
+ b = self.dls.one_batch()
+ self.learn._split(b)
+ self.writer.add_graph(self.model, *self.xb)
+
+ def after_batch(self):
+ self.writer.add_scalar('train_loss', self.smooth_loss, self.train_iter)
+ for i,h in enumerate(self.opt.hypers):
+ for k,v in h.items(): self.writer.add_scalar(f'{k}_{i}', v, self.train_iter)
+
+ def after_epoch(self):
+ for n,v in zip(self.recorder.metric_names[2:-1], self.recorder.log[2:-1]):
+ self.writer.add_scalar(n, v, self.train_iter)
+ if self.log_preds:
+ b = self.dls.valid.one_batch()
+ self.learn.one_batch(0, b)
+ preds = getcallable(self.loss_func, 'activation')(self.pred)
+ out = getcallable(self.loss_func, 'decodes')(preds)
+ x,y,its,outs = self.dls.valid.show_results(b, out, show=False, max_n=self.n_preds)
+ tensorboard_log(x, y, its, outs, self.writer, self.train_iter)
+ 
+ def before_validate(self):
+ if self.projector: self._setup_projector()
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 23
+class TensorBoardProjectorCallback(TensorBoardBaseCallback):
+ "Extracts and exports image featuers for tensorboard projector during inference"
+ def __init__(self, log_dir=None, layer=None):
+ super().__init__()
+ store_attr()
+ 
+ def before_fit(self):
+ self.run = not hasattr(self.learn, 'lr_finder') and hasattr(self, "gather_preds") and rank_distrib()==0
+ if not self.run: return
+ self._setup_writer()
+
+ def before_validate(self):
+ self._setup_projector()
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 25
+def _write_projector_embedding(learn, writer, feat):
+ lbls = [learn.dl.vocab[l] for l in feat['lbl']] if getattr(learn.dl, 'vocab', None) else None 
+ vecs = feat['vec'].squeeze()
+ writer.add_embedding(vecs, metadata=lbls, label_img=feat['img'], global_step=learn.train_iter)
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 26
+def _add_projector_features(learn, hook, feat):
+ img = _normalize_for_projector(learn.x)
+ first_epoch = True if learn.iter == 0 else False
+ feat['vec'] = hook.stored if first_epoch else torch.cat((feat['vec'], hook.stored),0)
+ feat['img'] = img if first_epoch else torch.cat((feat['img'], img),0)
+ if getattr(learn.dl, 'vocab', None):
+ feat['lbl'] = learn.y if first_epoch else torch.cat((feat['lbl'], learn.y),0)
+ return feat
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 27
+def _get_embeddings(model, layer):
+ layer = model[0].encoder if layer == None else layer
+ return layer.weight
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 28
+@typedispatch
+def _normalize_for_projector(x:TensorImage):
+ # normalize tensor to be between 0-1
+ img = x.clone()
+ sz = img.shape
+ img = img.view(x.size(0), -1)
+ img -= img.min(1, keepdim=True)[0]
+ img /= img.max(1, keepdim=True)[0]
+ img = img.view(*sz)
+ return img
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 29
+from ..text.all import LMLearner, TextLearner
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 30
+def projector_word_embeddings(learn=None, layer=None, vocab=None, limit=-1, start=0, log_dir=None):
+ "Extracts and exports word embeddings from language models embedding layers"
+ if not layer:
+ if isinstance(learn, LMLearner): layer = learn.model[0].encoder
+ elif isinstance(learn, TextLearner): layer = learn.model[0].module.encoder
+ emb = layer.weight
+ img = torch.full((len(emb),3,8,8), 0.7)
+ vocab = learn.dls.vocab[0] if vocab == None else vocab
+ vocab = list(map(lambda x: f'{x}_', vocab))
+ writer = SummaryWriter(log_dir=log_dir)
+ end = start + limit if limit >= 0 else -1
+ writer.add_embedding(emb[start:end], metadata=vocab[start:end], label_img=img[start:end])
+ writer.close()
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 32
+from ..vision.data import *
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 33
+@typedispatch
+def tensorboard_log(x:TensorImage, y: TensorCategory, samples, outs, writer, step):
+ fig,axs = get_grid(len(samples), return_fig=True)
+ for i in range(2):
+ axs = [b.show(ctx=c) for b,c in zip(samples.itemgot(i),axs)]
+ axs = [r.show(ctx=c, color='green' if b==r else 'red')
+ for b,r,c in zip(samples.itemgot(1),outs.itemgot(0),axs)]
+ writer.add_figure('Sample results', fig, step)
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 34
+from ..vision.core import TensorPoint,TensorBBox
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 35
+@typedispatch
+def tensorboard_log(x:TensorImage, y: TensorImageBase|TensorPoint|TensorBBox, samples, outs, writer, step):
+ fig,axs = get_grid(len(samples), return_fig=True, double=True)
+ for i in range(2):
+ axs[::2] = [b.show(ctx=c) for b,c in zip(samples.itemgot(i),axs[::2])]
+ for x in [samples,outs]:
+ axs[1::2] = [b.show(ctx=c) for b,c in zip(x.itemgot(0),axs[1::2])]
+ writer.add_figure('Sample results', fig, step)

fastai/callback/tracker.py

@@ -0,0 +1,139 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/17_callback.tracker.ipynb.
+
+# %% ../../nbs/17_callback.tracker.ipynb 2
+from __future__ import annotations
+from ..basics import *
+from .progress import *
+from .fp16 import MixedPrecision
+
+# %% auto 0
+__all__ = ['TerminateOnNaNCallback', 'TrackerCallback', 'EarlyStoppingCallback', 'SaveModelCallback', 'ReduceLROnPlateau']
+
+# %% ../../nbs/17_callback.tracker.ipynb 6
+class TerminateOnNaNCallback(Callback):
+ "A `Callback` that terminates training if loss is NaN."
+ order=-9
+ def after_batch(self):
+ "Test if `last_loss` is NaN and interrupts training."
+ if torch.isinf(self.loss) or torch.isnan(self.loss): raise CancelFitException
+
+# %% ../../nbs/17_callback.tracker.ipynb 10
+class TrackerCallback(Callback):
+ "A `Callback` that keeps track of the best value in `monitor`."
+ order,remove_on_fetch,_only_train_loop = 60,True,True
+ def __init__(self, 
+ monitor='valid_loss', # value (usually loss or metric) being monitored.
+ comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.
+ min_delta=0., # minimum delta between the last monitor value and the best monitor value.
+ reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
+ ):
+ if comp is None: comp = np.less if 'loss' in monitor or 'error' in monitor else np.greater
+ if comp == np.less: min_delta *= -1
+ self.monitor,self.comp,self.min_delta,self.reset_on_fit,self.best= monitor,comp,min_delta,reset_on_fit,None
+
+ def before_fit(self):
+ "Prepare the monitored value"
+ self.run = not hasattr(self, "lr_finder") and not hasattr(self, "gather_preds")
+ if self.reset_on_fit or self.best is None: self.best = float('inf') if self.comp == np.less else -float('inf')
+ assert self.monitor in self.recorder.metric_names[1:]
+ self.idx = list(self.recorder.metric_names[1:]).index(self.monitor)
+
+ def after_epoch(self):
+ "Compare the last value to the best up to now"
+ val = self.recorder.values[-1][self.idx]
+ if self.comp(val - self.min_delta, self.best): self.best,self.new_best = val,True
+ else: self.new_best = False
+
+ def after_fit(self): self.run=True
+
+# %% ../../nbs/17_callback.tracker.ipynb 19
+class EarlyStoppingCallback(TrackerCallback):
+ "A `TrackerCallback` that terminates training when monitored quantity stops improving."
+ order=TrackerCallback.order+3
+ def __init__(self, 
+ monitor='valid_loss', # value (usually loss or metric) being monitored.
+ comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.
+ min_delta=0., # minimum delta between the last monitor value and the best monitor value.
+ patience=1, # number of epochs to wait when training has not improved model.
+ reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
+ ):
+ super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)
+ self.patience = patience
+
+ def before_fit(self): self.wait = 0; super().before_fit()
+ def after_epoch(self):
+ "Compare the value monitored to its best score and maybe stop training."
+ super().after_epoch()
+ if self.new_best: self.wait = 0
+ else:
+ self.wait += 1
+ if self.wait >= self.patience:
+ print(f'No improvement since epoch {self.epoch-self.wait}: early stopping')
+ raise CancelFitException()
+
+# %% ../../nbs/17_callback.tracker.ipynb 26
+class SaveModelCallback(TrackerCallback):
+ "A `TrackerCallback` that saves the model's best during training and loads it at the end."
+ order = TrackerCallback.order+1
+ def __init__(self, 
+ monitor='valid_loss', # value (usually loss or metric) being monitored.
+ comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.
+ min_delta=0., # minimum delta between the last monitor value and the best monitor value.
+ fname='model', # model name to be used when saving model.
+ every_epoch=False, # if true, save model after every epoch; else save only when model is better than existing best.
+ at_end=False, # if true, save model when training ends; else load best model if there is only one saved model.
+ with_opt=False, # if true, save optimizer state (if any available) when saving model. 
+ reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
+ ):
+ super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)
+ assert not (every_epoch and at_end), "every_epoch and at_end cannot both be set to True"
+ # keep track of file path for loggers
+ self.last_saved_path = None
+ store_attr('fname,every_epoch,at_end,with_opt')
+
+ def _save(self, name): self.last_saved_path = self.learn.save(name, with_opt=self.with_opt)
+
+ def after_epoch(self):
+ "Compare the value monitored to its best score and save if best."
+ if self.every_epoch:
+ if (self.epoch%self.every_epoch) == 0: self._save(f'{self.fname}_{self.epoch}')
+ else: #every improvement
+ super().after_epoch()
+ if self.new_best:
+ print(f'Better model found at epoch {self.epoch} with {self.monitor} value: {self.best}.')
+ self._save(f'{self.fname}')
+
+ def after_fit(self, **kwargs):
+ "Load the best model."
+ if self.at_end: self._save(f'{self.fname}')
+ elif not self.every_epoch: self.learn.load(f'{self.fname}', with_opt=self.with_opt)
+
+# %% ../../nbs/17_callback.tracker.ipynb 30
+class ReduceLROnPlateau(TrackerCallback):
+ "A `TrackerCallback` that reduces learning rate when a metric has stopped improving."
+ order=TrackerCallback.order+2
+ def __init__(self, 
+ monitor='valid_loss', # value (usually loss or metric) being monitored.
+ comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.
+ min_delta=0., # minimum delta between the last monitor value and the best monitor value.
+ patience=1, # number of epochs to wait when training has not improved model.
+ factor=10., # the denominator to divide the learning rate by, when reducing the learning rate.
+ min_lr=0, # the minimum learning rate allowed; learning rate cannot be reduced below this minimum.
+ reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
+ ):
+ super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)
+ self.patience,self.factor,self.min_lr = patience,factor,min_lr
+
+ def before_fit(self): self.wait = 0; super().before_fit()
+ def after_epoch(self):
+ "Compare the value monitored to its best score and reduce LR by `factor` if no improvement."
+ super().after_epoch()
+ if self.new_best: self.wait = 0
+ else:
+ self.wait += 1
+ if self.wait >= self.patience:
+ old_lr = self.opt.hypers[-1]['lr']
+ for h in self.opt.hypers: h['lr'] = max(h['lr'] / self.factor, self.min_lr)
+ self.wait = 0
+ if self.opt.hypers[-1]["lr"] < old_lr:
+ print(f'Epoch {self.epoch}: reducing lr to {self.opt.hypers[-1]["lr"]}')