File size: 842 Bytes
d90b3a8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
"""Adapter to expose MegaBlocks package, if available."""
try:
import megablocks
except ImportError:
megablocks = None
def megablocks_is_available():
return megablocks is not None
def assert_megablocks_is_available():
assert (
megablocks_is_available()
), "MegaBlocks not available. Please run `pip install megablocks`."
moe = megablocks.layers.moe if megablocks_is_available() else None
dmoe = megablocks.layers.dmoe if megablocks_is_available() else None
arguments = megablocks.layers.arguments if megablocks_is_available() else None
def as_megablocks_args(neox_args):
import copy
tmp = copy.copy(neox_args)
args = arguments.from_megatron(tmp)
args.moe_lbl_in_fp32 = True
args.fp16 = neox_args.precision == "fp16"
args.moe_loss_weight = neox_args.moe_loss_coeff
return args
|