update

language_identification.md

@@ -0,0 +1,18 @@
+## Language Identification
+
+### lanid
+
+langid 识别 97 种语言。
+https://github.com/saffsd/langid.py
+
+原理：
+```text
+https://github.com/saffsd/langid.py/tree/master/langid/train
+
+1. 分词.
+2. 计算 `字符ngram` 或 `词ngram` 特征.
+3. 计算 item 的文档频率.
+4. 计算 IG weights 信息增益权重, 提取重要特征.
+4. 训练 NB (Naive Bayes) 概率模型, 即每个 item 对每个类型的概率贡献. 
+
+```

main.py

@@ -20,6 +20,11 @@ from project_settings import project_path, temp_directory
 
 def get_args():
  parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--language_identification_md_file",
+ default=(project_path / "language_identification.md").as_posix(),
+ type=str
+ )
  parser.add_argument(
  "--lang_id_examples_file",
  default=(project_path / "lang_id_examples.json").as_posix(),
@@ -47,12 +52,13 @@ def main():
  args = get_args()
 
  brief_description = """
- ## Language Identification
-
- langid 识别 97 种语言。
- https://github.com/saffsd/langid.py
+ ### Language Identification
  """
 
+ # description
+ with open(args.language_identification_md_file, "r", encoding="utf-8") as f:
+ description = f.read()
+
  # examples
  with open(args.lang_id_examples_file, "r", encoding="utf-8") as f:
  lang_id_examples = json.load(f)
@@ -101,6 +107,8 @@ def main():
  outputs=[lang_id_label, lang_id_prob],
  )
 
+ gr.Markdown(value=description)
+
  blocks.queue().launch(
  share=False if platform.system() == "Windows" else False,
  server_name="127.0.0.1" if platform.system() == "Windows" else "0.0.0.0",