Upload 6 files

normalize.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import re
+import regex
+
+from itertools import chain
+
+
+class MosesPunctNormalizer:
+ """
+ This is a Python port of the Moses punctuation normalizer from
+ https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/normalize-punctuation.perl
+ """
+
+ EXTRA_WHITESPACE = [ # lines 21 - 30
+ (r"\r", r""),
+ (r"\(", r" ("),
+ (r"\)", r") "),
+ (r" +", r" "),
+ (r"\) ([.!:?;,])", r")\g<1>"),
+ (r"\( ", r"("),
+ (r" \)", r")"),
+ (r"(\d) %", r"\g<1>%"),
+ (r" :", r":"),
+ (r" ;", r";"),
+ ]
+
+ NORMALIZE_UNICODE_IF_NOT_PENN = [(r"`", r"'"), (r"''", r' " ')] # lines 33 - 34
+
+ NORMALIZE_UNICODE = [ # lines 37 - 50
+ ("„", r'"'),
+ ("“", r'"'),
+ ("”", r'"'),
+ ("–", r"-"),
+ ("—", r" - "),
+ (r" +", r" "),
+ ("´", r"'"),
+ ("([a-zA-Z])‘([a-zA-Z])", r"\g<1>'\g<2>"),
+ ("([a-zA-Z])’([a-zA-Z])", r"\g<1>'\g<2>"),
+ ("‘", r"'"),
+ ("‚", r"'"),
+ ("’", r"'"),
+ (r"''", r'"'),
+ ("´´", r'"'),
+ ("…", r"..."),
+ ]
+
+ FRENCH_QUOTES = [ # lines 52 - 57
+ ("\u00A0«\u00A0", r'"'),
+ ("«\u00A0", r'"'),
+ ("«", r'"'),
+ ("\u00A0»\u00A0", r'"'),
+ ("\u00A0»", r'"'),
+ ("»", r'"'),
+ ]
+
+ HANDLE_PSEUDO_SPACES = [ # lines 59 - 67
+ ("\u00A0%", r"%"),
+ ("nº\u00A0", "nº "),
+ ("\u00A0:", r":"),
+ ("\u00A0ºC", " ºC"),
+ ("\u00A0cm", r" cm"),
+ ("\u00A0\\?", "?"),
+ ("\u00A0\\!", "!"),
+ ("\u00A0;", r";"),
+ (",\u00A0", r", "),
+ (r" +", r" "),
+ ]
+
+ EN_QUOTATION_FOLLOWED_BY_COMMA = [(r'"([,.]+)', r'\g<1>"')]
+
+ DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA = [
+ (r',"', r'",'),
+ (r'(\.+)"(\s*[^<])', r'"\g<1>\g<2>'), # don't fix period at end of sentence
+ ]
+
+ DE_ES_CZ_CS_FR = [
+ ("(\\d)\u00A0(\\d)", r"\g<1>,\g<2>"),
+ ]
+
+ OTHER = [
+ ("(\\d)\u00A0(\\d)", r"\g<1>.\g<2>"),
+ ]
+
+ # Regex substitutions from replace-unicode-punctuation.perl
+ # https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
+ REPLACE_UNICODE_PUNCTUATION = [
+ ("，", ","),
+ (r"。\s*", ". "),
+ ("、", ","),
+ ("”", '"'),
+ ("“", '"'),
+ ("∶", ":"),
+ ("：", ":"),
+ ("？", "?"),
+ ("《", '"'),
+ ("》", '"'),
+ ("）", ")"),
+ ("！", "!"),
+ ("（", "("),
+ ("；", ";"),
+ ("」", '"'),
+ ("「", '"'),
+ ("０", "0"),
+ ("１", "1"),
+ ("２", "2"),
+ ("３", "3"),
+ ("４", "4"),
+ ("５", "5"),
+ ("６", "6"),
+ ("７", "7"),
+ ("８", "8"),
+ ("９", "9"),
+ (r"．\s*", ". "),
+ ("～", "~"),
+ ("’", "'"),
+ ("…", "..."),
+ ("━", "-"),
+ ("〈", "<"),
+ ("〉", ">"),
+ ("【", "["),
+ ("】", "]"),
+ ("％", "%"),
+ ]
+
+ def __init__(
+ self,
+ lang="en",
+ penn=True,
+ norm_quote_commas=True,
+ norm_numbers=True,
+ pre_replace_unicode_punct=False,
+ post_remove_control_chars=False,
+ ):
+ """
+ :param language: The two-letter language code.
+ :type lang: str
+ :param penn: Normalize Penn Treebank style quotations.
+ :type penn: bool
+ :param norm_quote_commas: Normalize quotations and commas
+ :type norm_quote_commas: bool
+ :param norm_numbers: Normalize numbers
+ :type norm_numbers: bool
+ """
+ self.substitutions = [
+ self.EXTRA_WHITESPACE,
+ self.NORMALIZE_UNICODE,
+ self.FRENCH_QUOTES,
+ self.HANDLE_PSEUDO_SPACES,
+ ]
+
+ if penn: # Adds the penn substitutions after extra_whitespace regexes.
+ self.substitutions.insert(1, self.NORMALIZE_UNICODE_IF_NOT_PENN)
+
+ if norm_quote_commas:
+ if lang == "en":
+ self.substitutions.append(self.EN_QUOTATION_FOLLOWED_BY_COMMA)
+ elif lang in ["de", "es", "fr"]:
+ self.substitutions.append(self.DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA)
+
+ if norm_numbers:
+ if lang in ["de", "es", "cz", "cs", "fr"]:
+ self.substitutions.append(self.DE_ES_CZ_CS_FR)
+ else:
+ self.substitutions.append(self.OTHER)
+
+ self.substitutions = list(chain(*self.substitutions))
+
+ self.pre_replace_unicode_punct = pre_replace_unicode_punct
+ self.post_remove_control_chars = post_remove_control_chars
+
+ def normalize(self, text):
+ """
+ Returns a string with normalized punctuation.
+ """
+ # Optionally, replace unicode puncts BEFORE normalization.
+ if self.pre_replace_unicode_punct:
+ text = self.replace_unicode_punct(text)
+
+ # Actual normalization.
+ for regexp, substitution in self.substitutions:
+ # print(regexp, substitution)
+ text = re.sub(regexp, substitution, str(text))
+ # print(text)
+
+ # Optionally, replace unicode puncts BEFORE normalization.
+ if self.post_remove_control_chars:
+ text = self.remove_control_chars(text)
+
+ return text.strip()
+
+ def replace_unicode_punct(self, text):
+ for regexp, substitution in self.REPLACE_UNICODE_PUNCTUATION:
+ text = re.sub(regexp, substitution, str(text))
+ return text
+
+ def remove_control_chars(self, text):
+ return regex.sub(r"\p{C}", "", text)

tokenization_korscideberta.py

@@ -0,0 +1,356 @@
+# coding=utf-8
+# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team and Jangwon Park
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Tokenization classes for KoBERT model """
+
+
+import logging
+import os
+import unicodedata
+from shutil import copyfile
+
+from transformers import PreTrainedTokenizer
+
+#2023. 7. 28. 형태소 분리(Mecab), 유니코드 정규화 추가
+from konlpy.tag import Mecab
+from unicode import join_jamos
+from normalize import MosesPunctNormalizer
+nor = MosesPunctNormalizer()
+
+# 유니코드 한글 시작 : 44032, 끝 : 55199
+BASE_CODE, CHOSUNG, JUNGSUNG = 44032, 588, 28
+# 초성 리스트. 0 ~ 18
+CHOSUNG_LIST = ['ㄱ', 'ㄲ', 'ㄴ', 'ㄷ', 'ㄸ', 'ㄹ', 'ㅁ', 'ㅂ', 'ㅃ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅉ', 'ㅊ', 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ']
+# 중성 리스트. 0 ~ 20
+JUNGSUNG_LIST = ['ㅏ', 'ㅐ', 'ㅑ', 'ㅒ', 'ㅓ', 'ㅔ', 'ㅕ', 'ㅖ', 'ㅗ', 'ㅘ', 'ㅙ', 'ㅚ', 'ㅛ', 'ㅜ', 'ㅝ', 'ㅞ', 'ㅟ', 'ㅠ', 'ㅡ', 'ㅢ', 'ㅣ']
+# 종성 리스트. 0 ~ 27 + 1(1개 없음)
+JONGSUNG_LIST = [' ', 'ㄱ', 'ㄲ', 'ㄳ', 'ㄴ', 'ㄵ', 'ㄶ', 'ㄷ', 'ㄹ', 'ㄺ', 'ㄻ', 'ㄼ', 'ㄽ', 'ㄾ', 'ㄿ', 'ㅀ', 'ㅁ', 'ㅂ', 'ㅄ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅊ', 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ']
+def splitjamo(string):
+ sp_list = list(string)
+ result = []
+ for keyword in sp_list:
+ # 한글 여부 check 후 분리
+ if re.match('.*[ㄱ-ㅎㅏ-ㅣ가-힣]+.*', keyword) is not None:
+ # 초성
+ char_code = ord(keyword) - BASE_CODE
+ char1 = int(char_code / CHOSUNG)
+ try:
+ result.append(CHOSUNG_LIST[char1])
+ except:
+ return string
+ #print("Err: "+str(char1))
+ # 중성
+ char2 = int((char_code - (CHOSUNG * char1)) / JUNGSUNG)
+ result.append(JUNGSUNG_LIST[char2])
+ # 종성
+ char3 = int((char_code - (CHOSUNG * char1) - (JUNGSUNG * char2)))
+ result.append(JONGSUNG_LIST[char3])
+ else:
+ result.append(keyword)
+ return result
+def has_coda(word):
+ return (ord(word[-1]) -44032)%28==0
+def _replace_unicode(line):
+ if(line==None):
+ return ""
+ line = line.replace("—",'-').replace("―","-").replace("–","-").replace("＂",'"').replace("＇","'").replace("‹","<").replace("›",">").replace("‚","'").replace("‛","'").replace("„",'"').replace("‟",'"').replace("«",'<').replace("»",'>').replace("˝",'"').replace("（",'(').replace("）",')').replace("『",'"').replace("』",'"').replace("“",'"').replace("”",'"').replace("‘","'").replace("’","'").replace("《","<").replace("》",">").replace("〈","<").replace("〉",">").replace("「","'").replace("」","'").replace("【","[").replace("】","]").replace("〔","[").replace("〕","]").replace("［","[").replace("］","]").replace("｛","{").replace("｝","}")
+ line=nor.replace_unicode_punct(line)
+ return line
+def _mecab(line):
+ mecab = Mecab()
+ #참고: VV동사 VA형용사 VX보조 용언 VCP긍정 지정사 VCN부정 지정사 JKS주격 조사 JKC보격 조사, … XSN명사 파생 접미사 XSV동사 파생 접미사 XSA형용사 파생 접미사 EP선어말 어미 EF종결 어미 EC연결 어미 ETN명사형 전성 어미 ETM관형형 전성 어미
+ 
+ pdoc = []
+ morphs = []
+ 
+ poss = mecab.pos(line)
+ for pos in poss:
+ morphs.append(pos[0])
+ '''
+ pdoc.append(" ".join(morphs))
+ return pdoc
+ '''
+ return " ".join(morphs)
+
+logger = logging.getLogger(__name__)
+
+VOCAB_FILES_NAMES = {
+ "vocab_file": "spm.model",
+ "vocab_txt": "vocab.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+ "vocab_file": {
+ "monologg/kobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert/tokenizer_78b3253a26.model",
+ "monologg/kobert-lm": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert-lm/tokenizer_78b3253a26.model",
+ "monologg/distilkobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/distilkobert/tokenizer_78b3253a26.model",
+ },
+ "vocab_txt": {
+ "monologg/kobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert/vocab.txt",
+ "monologg/kobert-lm": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert-lm/vocab.txt",
+ "monologg/distilkobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/distilkobert/vocab.txt",
+ },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+ "monologg/kobert": 512,
+ "monologg/kobert-lm": 512,
+ "monologg/distilkobert": 512,
+}
+
+PRETRAINED_INIT_CONFIGURATION = {
+ "monologg/kobert": {"do_lower_case": False},
+ "monologg/kobert-lm": {"do_lower_case": False},
+ "monologg/distilkobert": {"do_lower_case": False},
+}
+
+SPIECE_UNDERLINE = "▁"
+
+
+class DebertaV2Tokenizer(PreTrainedTokenizer):
+ """
+ SentencePiece based tokenizer. Peculiarities:
+ - requires `SentencePiece <https://github.com/google/sentencepiece>`_
+ """
+
+ vocab_files_names = VOCAB_FILES_NAMES
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+ pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+ def __init__(
+ self,
+ vocab_file,
+ vocab_txt,
+ do_lower_case=False,
+ remove_space=True,
+ keep_accents=False,
+ unk_token="<unk>",
+ sep_token="<s>",
+ pad_token="<pad>",
+ cls_token="<cls>",
+ mask_token="<mask>",
+ **kwargs,
+ ):
+ super().__init__(
+ unk_token="<unk>",
+ sep_token=sep_token,
+ pad_token=pad_token,
+ cls_token=cls_token,
+ mask_token=mask_token,
+ **kwargs,
+ )
+
+ # Build vocab
+ self.token2idx = dict()
+ self.idx2token = []
+ with open(vocab_txt, "r", encoding="utf-8") as f:
+ for idx, token in enumerate(f):
+ token = token.strip()
+ self.token2idx[token] = idx
+ self.idx2token.append(token)
+
+ try:
+ import sentencepiece as spm
+ except ImportError:
+ logger.warning(
+ "You need to install SentencePiece to use KoBertTokenizer: https://github.com/google/sentencepiece"
+ "pip install sentencepiece"
+ )
+
+ self.do_lower_case = do_lower_case
+ self.remove_space = remove_space
+ self.keep_accents = keep_accents
+ self.vocab_file = vocab_file
+ self.vocab_txt = vocab_txt
+
+ self.sp_model = spm.SentencePieceProcessor()
+ self.sp_model.Load(vocab_file)
+
+ @property
+ def vocab_size(self):
+ return len(self.idx2token)
+
+ def get_vocab(self):
+ return dict(self.token2idx, **self.added_tokens_encoder)
+
+ def __getstate__(self):
+ state = self.__dict__.copy()
+ state["sp_model"] = None
+ return state
+
+ def __setstate__(self, d):
+ self.__dict__ = d
+ try:
+ import sentencepiece as spm
+ except ImportError:
+ logger.warning(
+ "You need to install SentencePiece to use KoBertTokenizer: https://github.com/google/sentencepiece"
+ "pip install sentencepiece"
+ )
+ self.sp_model = spm.SentencePieceProcessor()
+ self.sp_model.Load(self.vocab_file)
+
+ def preprocess_text(self, inputs):
+ if self.remove_space:
+ outputs = " ".join(inputs.strip().split())
+ else:
+ outputs = inputs
+ outputs = outputs.replace("``", '"').replace("''", '"')
+
+ if not self.keep_accents:
+ outputs = unicodedata.normalize("NFKD", outputs)
+ outputs = "".join([c for c in outputs if not unicodedata.combining(c)])
+ if self.do_lower_case:
+ outputs = outputs.lower()
+
+ return outputs
+
+ def _tokenize(self, text):
+ """Tokenize a string."""
+ text = self.preprocess_text(text)
+ #print('text: '+text)
+ #logger.info("text ({}) ".format(text))
+ text = _replace_unicode(text) #유니코드 정규화
+ text = _mecab(text) #형태소 분리
+ #print('text: '+str(text))
+ #logger.info("text ({}) ".format(text))
+ pieces = self.sp_model.encode(text, out_type=str)
+ new_pieces = []
+ for piece in pieces:
+ if len(piece) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
+ cur_pieces = self.sp_model.EncodeAsPieces(piece[:-1].replace(SPIECE_UNDERLINE, ""))
+ if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
+ if len(cur_pieces[0]) == 1:
+ cur_pieces = cur_pieces[1:]
+ else:
+ cur_pieces[0] = cur_pieces[0][1:]
+ cur_pieces.append(piece[-1])
+ new_pieces.extend(cur_pieces)
+ else:
+ new_pieces.append(piece)
+ '''
+ return_pieces = []
+ for n in new_pieces:
+ if(isinstance(n,list)):
+ for nn in n:
+ return_pieces.append(nn)
+ else:
+ return_pieces.append(n)
+ return return_pieces
+ '''
+ return new_pieces
+ 
+
+ def _convert_token_to_id(self, token):
+ """ Converts a token (str/unicode) in an id using the vocab. """
+ return self.token2idx.get(token, self.token2idx[self.unk_token])
+
+ def _convert_id_to_token(self, index):
+ """Converts an index (integer) in a token (string/unicode) using the vocab."""
+ return self.idx2token[index]
+
+ def convert_tokens_to_string(self, tokens):
+ """Converts a sequence of tokens (strings for sub-words) in a single string."""
+ out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
+ return out_string
+
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+ """
+ Build model inputs from a sequence or a pair of sequence for sequence classification tasks
+ by concatenating and adding special tokens.
+ A KoBERT sequence has the following format:
+ single sequence: [CLS] X [SEP]
+ pair of sequences: [CLS] A [SEP] B [SEP]
+ """
+ if token_ids_1 is None:
+ return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
+ cls = [self.cls_token_id]
+ sep = [self.sep_token_id]
+ return cls + token_ids_0 + sep + token_ids_1 + sep
+
+ def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
+ """
+ Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+ special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
+ Args:
+ token_ids_0: list of ids (must not contain special tokens)
+ token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids
+ for sequence pairs
+ already_has_special_tokens: (default False) Set to True if the token list is already formated with
+ special tokens for the model
+ Returns:
+ A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+ """
+
+ if already_has_special_tokens:
+ if token_ids_1 is not None:
+ raise ValueError(
+ "You should not supply a second sequence if the provided sequence of "
+ "ids is already formated with special tokens for the model."
+ )
+ return list(
+ map(
+ lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0,
+ token_ids_0,
+ )
+ )
+
+ if token_ids_1 is not None:
+ return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
+ return [1] + ([0] * len(token_ids_0)) + [1]
+
+ def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+ """
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
+ A KoBERT sequence pair mask has the following format:
+ 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1
+ | first sequence | second sequence
+ if token_ids_1 is None, only returns the first portion of the mask (0's).
+ """
+ sep = [self.sep_token_id]
+ cls = [self.cls_token_id]
+ if token_ids_1 is None:
+ return len(cls + token_ids_0 + sep) * [0]
+ return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]
+
+ def save_vocabulary(self, save_directory):
+ """Save the sentencepiece vocabulary (copy original file) and special tokens file
+ to a directory.
+ """
+ if not os.path.isdir(save_directory):
+ logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
+ return
+
+ # 1. Save sentencepiece model
+ out_vocab_model = os.path.join(save_directory, VOCAB_FILES_NAMES["vocab_file"])
+
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_model):
+ copyfile(self.vocab_file, out_vocab_model)
+
+ # 2. Save vocab.txt
+ index = 0
+ out_vocab_txt = os.path.join(save_directory, VOCAB_FILES_NAMES["vocab_txt"])
+ with open(out_vocab_txt, "w", encoding="utf-8") as writer:
+ for token, token_index in sorted(self.token2idx.items(), key=lambda kv: kv[1]):
+ if index != token_index:
+ logger.warning(
+ "Saving vocabulary to {}: vocabulary indices are not consecutive."
+ " Please check that the vocabulary is not corrupted!".format(out_vocab_txt)
+ )
+ index = token_index
+ writer.write(token + "\n")
+ index += 1
+
+ return out_vocab_model, out_vocab_txt

tokenizer_config.json

@@ -0,0 +1,4 @@
+{
+ "do_lower_case": false,
+ "vocab_type": "spm"
+}

unicode.py

@@ -0,0 +1,279 @@
+__all__ = ["split_syllable_char", "split_syllables",
+ "join_jamos", "join_jamos_char",
+ "CHAR_INITIALS", "CHAR_MEDIALS", "CHAR_FINALS"]
+
+import itertools
+
+INITIAL = 0x001
+MEDIAL = 0x010
+FINAL = 0x100
+CHAR_LISTS = {
+ INITIAL: list(map(chr, [
+ 0x3131, 0x3132, 0x3134, 0x3137, 0x3138, 0x3139,
+ 0x3141, 0x3142, 0x3143, 0x3145, 0x3146, 0x3147,
+ 0x3148, 0x3149, 0x314a, 0x314b, 0x314c, 0x314d,
+ 0x314e
+ ])),
+ MEDIAL: list(map(chr, [
+ 0x314f, 0x3150, 0x3151, 0x3152, 0x3153, 0x3154,
+ 0x3155, 0x3156, 0x3157, 0x3158, 0x3159, 0x315a,
+ 0x315b, 0x315c, 0x315d, 0x315e, 0x315f, 0x3160,
+ 0x3161, 0x3162, 0x3163
+ ])),
+ FINAL: list(map(chr, [
+ 0x3131, 0x3132, 0x3133, 0x3134, 0x3135, 0x3136,
+ 0x3137, 0x3139, 0x313a, 0x313b, 0x313c, 0x313d,
+ 0x313e, 0x313f, 0x3140, 0x3141, 0x3142, 0x3144,
+ 0x3145, 0x3146, 0x3147, 0x3148, 0x314a, 0x314b,
+ 0x314c, 0x314d, 0x314e
+ ]))
+}
+CHAR_INITIALS = CHAR_LISTS[INITIAL]
+CHAR_MEDIALS = CHAR_LISTS[MEDIAL]
+CHAR_FINALS = CHAR_LISTS[FINAL]
+CHAR_SETS = {k: set(v) for k, v in CHAR_LISTS.items()}
+CHARSET = set(itertools.chain(*CHAR_SETS.values()))
+CHAR_INDICES = {k: {c: i for i, c in enumerate(v)}
+ for k, v in CHAR_LISTS.items()}
+
+
+def is_hangul_syllable(c):
+ return 0xac00 <= ord(c) <= 0xd7a3 # Hangul Syllables
+
+
+def is_hangul_jamo(c):
+ return 0x1100 <= ord(c) <= 0x11ff # Hangul Jamo
+
+
+def is_hangul_compat_jamo(c):
+ return 0x3130 <= ord(c) <= 0x318f # Hangul Compatibility Jamo
+
+
+def is_hangul_jamo_exta(c):
+ return 0xa960 <= ord(c) <= 0xa97f # Hangul Jamo Extended-A
+
+
+def is_hangul_jamo_extb(c):
+ return 0xd7b0 <= ord(c) <= 0xd7ff # Hangul Jamo Extended-B
+
+
+def is_hangul(c):
+ return (is_hangul_syllable(c) or
+ is_hangul_jamo(c) or
+ is_hangul_compat_jamo(c) or
+ is_hangul_jamo_exta(c) or
+ is_hangul_jamo_extb(c))
+
+
+def is_supported_hangul(c):
+ return is_hangul_syllable(c) or is_hangul_compat_jamo(c)
+
+
+def check_hangul(c, jamo_only=False):
+ if not ((jamo_only or is_hangul_compat_jamo(c)) or is_supported_hangul(c)):
+ raise ValueError(f"'{c}' is not a supported hangul character. "
+ f"'Hangul Syllables' (0xac00 ~ 0xd7a3) and "
+ f"'Hangul Compatibility Jamos' (0x3130 ~ 0x318f) are "
+ f"supported at the moment.")
+
+
+def get_jamo_type(c):
+ check_hangul(c)
+ assert is_hangul_compat_jamo(c), f"not a jamo: {ord(c):x}"
+ return sum(t for t, s in CHAR_SETS.items() if c in s)
+
+
+def split_syllable_char(c):
+ """
+ Splits a given korean syllable into its components. Each component is
+ represented by Unicode in 'Hangul Compatibility Jamo' range.
+
+ Arguments:
+ c: A Korean character.
+
+ Returns:
+ A triple (initial, medial, final) of Hangul Compatibility Jamos.
+ If no jamo corresponds to a position, `None` is returned there.
+
+ Example:
+ >>> split_syllable_char("안")
+ ("ㅇ", "ㅏ", "ㄴ")
+ >>> split_syllable_char("고")
+ ("ㄱ", "ㅗ", None)
+ >>> split_syllable_char("ㅗ")
+ (None, "ㅗ", None)
+ >>> split_syllable_char("ㅇ")
+ ("ㅇ", None, None)
+ """
+ check_hangul(c)
+ if len(c) != 1:
+ raise ValueError("Input string must have exactly one character.")
+
+ init, med, final = None, None, None
+ if is_hangul_syllable(c):
+ offset = ord(c) - 0xac00
+ x = (offset - offset % 28) // 28
+ init, med, final = x // 21, x % 21, offset % 28
+ if not final:
+ final = None
+ else:
+ final -= 1
+ else:
+ pos = get_jamo_type(c)
+ if pos & INITIAL == INITIAL:
+ pos = INITIAL
+ elif pos & MEDIAL == MEDIAL:
+ pos = MEDIAL
+ elif pos & FINAL == FINAL:
+ pos = FINAL
+ idx = CHAR_INDICES[pos][c]
+ if pos == INITIAL:
+ init = idx
+ elif pos == MEDIAL:
+ med = idx
+ else:
+ final = idx
+ return tuple(CHAR_LISTS[pos][idx] if idx is not None else None
+ for pos, idx in
+ zip([INITIAL, MEDIAL, FINAL], [init, med, final]))
+
+
+def split_syllables(s, ignore_err=True, pad=None):
+ """
+ Performs syllable-split on a string.
+
+ Arguments:
+ s (str): A string (possibly mixed with non-Hangul characters).
+ ignore_err (bool): If set False, it ensures that all characters in
+ the string are Hangul-splittable and throws a ValueError otherwise.
+ (default: True)
+ pad (str): Pad empty jamo positions (initial, medial, or final) with
+ `pad` character. This is useful for cases where fixed-length
+ strings are needed. (default: None)
+
+ Returns:
+ Hangul-split string
+
+ Example:
+ >>> split_syllables("안녕하세요")
+ "ㅇㅏㄴㄴㅕㅇ��ㅏㅅㅔㅇㅛ"
+ >>> split_syllables("안녕하세요~~", ignore_err=False)
+ ValueError: encountered an unsupported character: ~ (0x7e)
+ >>> split_syllables("안녕하세요ㅛ", pad="x")
+ 'ㅇㅏㄴㄴㅕㅇㅎㅏxㅅㅔxㅇㅛxxㅛx'
+ """
+
+ def try_split(c):
+ try:
+ return split_syllable_char(c)
+ except ValueError:
+ if ignore_err:
+ return (c,)
+ raise ValueError(f"encountered an unsupported character: "
+ f"{c} (0x{ord(c):x})")
+
+ s = map(try_split, s)
+ if pad is not None:
+ tuples = map(lambda x: tuple(pad if y is None else y for y in x), s)
+ else:
+ tuples = map(lambda x: filter(None, x), s)
+ return "".join(itertools.chain(*tuples))
+
+
+def join_jamos_char(init, med, final=None):
+ """
+ Combines jamos into a single syllable.
+
+ Arguments:
+ init (str): Initial jao.
+ med (str): Medial jamo.
+ final (str): Final jamo. If not supplied, the final syllable is made
+ without the final. (default: None)
+
+ Returns:
+ A Korean syllable.
+ """
+ chars = (init, med, final)
+ for c in filter(None, chars):
+ check_hangul(c, jamo_only=True)
+
+ idx = tuple(CHAR_INDICES[pos][c] if c is not None else c
+ for pos, c in zip((INITIAL, MEDIAL, FINAL), chars))
+ init_idx, med_idx, final_idx = idx
+ # final index must be shifted once as
+ # final index with 0 points to syllables without final
+ final_idx = 0 if final_idx is None else final_idx + 1
+ return chr(0xac00 + 28 * 21 * init_idx + 28 * med_idx + final_idx)
+
+
+def join_jamos(s, ignore_err=True):
+ """
+ Combines a sequence of jamos to produce a sequence of syllables.
+
+ Arguments:
+ s (str): A string (possible mixed with non-jamo characters).
+ ignore_err (bool): If set False, it will ensure that all characters
+ will be consumed for the making of syllables. It will throw a
+ ValueError when it fails to do so. (default: True)
+
+ Returns:
+ A string
+
+ Example:
+ >>> join_jamos("ㅇㅏㄴㄴㅕㅇㅎㅏㅅㅔㅇㅛ")
+ "안녕하세요"
+ >>> join_jamos("ㅇㅏㄴㄴㄴㅕㅇㅎㅏㅅㅔㅇㅛ")
+ "안ㄴ녕하세요"
+ >>> join_jamos()
+ """
+ last_t = 0
+ queue = []
+ new_string = ""
+
+ def flush(n=0):
+ new_queue = []
+ while len(queue) > n:
+ new_queue.append(queue.pop())
+ if len(new_queue) == 1:
+ if not ignore_err:
+ raise ValueError(f"invalid jamo character: {new_queue[0]}")
+ result = new_queue[0]
+ elif len(new_queue) >= 2:
+ try:
+ result = join_jamos_char(*new_queue)
+ except (ValueError, KeyError):
+ # Invalid jamo combination
+ if not ignore_err:
+ raise ValueError(f"invalid jamo characters: {new_queue}")
+ result = "".join(new_queue)
+ else:
+ result = None
+ return result
+
+ for c in s:
+ if c not in CHARSET:
+ if queue:
+ new_c = flush() + c
+ else:
+ new_c = c
+ last_t = 0
+ else:
+ t = get_jamo_type(c)
+ new_c = None
+ if t & FINAL == FINAL:
+ if not (last_t == MEDIAL):
+ new_c = flush()
+ elif t == INITIAL:
+ new_c = flush()
+ elif t == MEDIAL:
+ if last_t & INITIAL == INITIAL:
+ new_c = flush(1)
+ else:
+ new_c = flush()
+ last_t = t
+ queue.insert(0, c)
+ if new_c:
+ new_string += new_c
+ if queue:
+ new_string += flush()
+ return new_string