Hello everyone

.gitignore

@@ -0,0 +1,886 @@
+### VirtualEnv template
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+.Python
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+.venv
+pip-selfcheck.json
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+### JupyterNotebooks template
+# gitignore template for Jupyter Notebooks
+# website: http://jupyter.org/
+
+.ipynb_checkpoints
+*/.ipynb_checkpoints/*
+
+# IPython
+profile_default/
+ipython_config.py
+
+# Remove previous ipynb_checkpoints
+#   git rm -r .ipynb_checkpoints/
+
+### Eclipse template
+.metadata
+bin/
+tmp/
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.settings/
+.loadpath
+.recommenders
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# PyDev specific (Python IDE for Eclipse)
+*.pydevproject
+
+# CDT-specific (C/C++ Development Tooling)
+.cproject
+
+# CDT- autotools
+.autotools
+
+# Java annotation processor (APT)
+.factorypath
+
+# PDT-specific (PHP Development Tools)
+.buildpath
+
+# sbteclipse plugin
+.target
+
+# Tern plugin
+.tern-project
+
+# TeXlipse plugin
+.texlipse
+
+# STS (Spring Tool Suite)
+.springBeans
+
+# Code Recommenders
+.recommenders/
+
+# Annotation Processing
+.apt_generated/
+.apt_generated_test/
+
+# Scala IDE specific (Scala & Java development for Eclipse)
+.cache-main
+.scala_dependencies
+.worksheet
+
+# Uncomment this line if you wish to ignore the project description file.
+# Typically, this file would be tracked if it contains build/dependency configurations:
+#.project
+
+### VisualStudio template
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Mono auto generated files
+mono_crash.*
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Ww][Ii][Nn]32/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+[Ll]ogs/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUnit
+*.VisualState.xml
+TestResult.xml
+nunit-*.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.obj
+*.iobj
+*.pch
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*_wpftmp.csproj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# NuGet Symbol Packages
+*.snupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+*.appxbundle
+*.appxupload
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
+
+### TeX template
+## Core latex/pdflatex auxiliary files:
+*.aux
+*.lof
+*.log
+*.lot
+*.fls
+*.out
+*.toc
+*.fmt
+*.fot
+*.cb
+*.cb2
+.*.lb
+
+## Intermediate documents:
+*.dvi
+*.xdv
+*-converted-to.*
+# these rules might exclude image files for figures etc.
+# *.ps
+# *.eps
+# *.pdf
+
+## Generated if empty string is given at "Please type another file name for output:"
+.pdf
+
+## Bibliography auxiliary files (bibtex/biblatex/biber):
+*.bbl
+*.bcf
+*.blg
+*-blx.aux
+*-blx.bib
+*.run.xml
+
+## Build tool auxiliary files:
+*.fdb_latexmk
+*.synctex
+*.synctex(busy)
+*.synctex.gz
+*.synctex.gz(busy)
+*.pdfsync
+
+## Build tool directories for auxiliary files
+# latexrun
+latex.out/
+
+## Auxiliary and intermediate files from other packages:
+# algorithms
+*.alg
+*.loa
+
+# achemso
+acs-*.bib
+
+# amsthm
+*.thm
+
+# beamer
+*.nav
+*.pre
+*.snm
+*.vrb
+
+# changes
+*.soc
+
+# comment
+*.cut
+
+# cprotect
+*.cpt
+
+# elsarticle (documentclass of Elsevier journals)
+*.spl
+
+# endnotes
+*.ent
+
+# fixme
+*.lox
+
+# feynmf/feynmp
+*.mf
+*.mp
+*.t[1-9]
+*.t[1-9][0-9]
+*.tfm
+
+#(r)(e)ledmac/(r)(e)ledpar
+*.end
+*.?end
+*.[1-9]
+*.[1-9][0-9]
+*.[1-9][0-9][0-9]
+*.[1-9]R
+*.[1-9][0-9]R
+*.[1-9][0-9][0-9]R
+*.eledsec[1-9]
+*.eledsec[1-9]R
+*.eledsec[1-9][0-9]
+*.eledsec[1-9][0-9]R
+*.eledsec[1-9][0-9][0-9]
+*.eledsec[1-9][0-9][0-9]R
+
+# glossaries
+*.acn
+*.acr
+*.glg
+*.glo
+*.gls
+*.glsdefs
+*.lzo
+*.lzs
+
+# uncomment this for glossaries-extra (will ignore makeindex's style files!)
+# *.ist
+
+# gnuplottex
+*-gnuplottex-*
+
+# gregoriotex
+*.gaux
+*.gtex
+
+# htlatex
+*.4ct
+*.4tc
+*.idv
+*.lg
+*.trc
+*.xref
+
+# hyperref
+*.brf
+
+# knitr
+*-concordance.tex
+# TODO Uncomment the next line if you use knitr and want to ignore its generated tikz files
+# *.tikz
+*-tikzDictionary
+
+# listings
+*.lol
+
+# luatexja-ruby
+*.ltjruby
+
+# makeidx
+*.idx
+*.ilg
+*.ind
+
+# minitoc
+*.maf
+*.mlf
+*.mlt
+*.mtc[0-9]*
+*.slf[0-9]*
+*.slt[0-9]*
+*.stc[0-9]*
+
+# minted
+_minted*
+*.pyg
+
+# morewrites
+*.mw
+
+# nomencl
+*.nlg
+*.nlo
+*.nls
+
+# pax
+*.pax
+
+# pdfpcnotes
+*.pdfpc
+
+# sagetex
+*.sagetex.sage
+*.sagetex.py
+*.sagetex.scmd
+
+# scrwfile
+*.wrt
+
+# sympy
+*.sout
+*.sympy
+sympy-plots-for-*.tex/
+
+# pdfcomment
+*.upa
+*.upb
+
+# pythontex
+*.pytxcode
+pythontex-files-*/
+
+# tcolorbox
+*.listing
+
+# thmtools
+*.loe
+
+# TikZ & PGF
+*.dpth
+*.md5
+*.auxlock
+
+# todonotes
+*.tdo
+
+# vhistory
+*.hst
+*.ver
+
+# easy-todo
+*.lod
+
+# xcolor
+*.xcp
+
+# xmpincl
+*.xmpi
+
+# xindy
+*.xdy
+
+# xypic precompiled matrices and outlines
+*.xyc
+*.xyd
+
+# endfloat
+*.ttt
+*.fff
+
+# Latexian
+TSWLatexianTemp*
+
+## Editors:
+# WinEdt
+*.bak
+*.sav
+
+# Texpad
+.texpadtmp
+
+# LyX
+*.lyx~
+
+# Kile
+*.backup
+
+# gummi
+.*.swp
+
+# KBibTeX
+*~[0-9]*
+
+# TeXnicCenter
+*.tps
+
+# auto folder when using emacs and auctex
+./auto/*
+*.el
+
+# expex forward references with \gathertags
+*-tags.tex
+
+# standalone packages
+*.sta
+
+# Makeindex log files
+*.lpz
+
+# xwatermark package
+*.xwm
+
+# REVTeX puts footnotes in the bibliography by default, unless the nofootinbib
+# option is specified. Footnotes are the stored in a file with suffix Notes.bib.
+# Uncomment the next line to have this generated file ignored.
+#*Notes.bib
+
+# CUSTOM
+.idea
+.DS_store
+.DS_Store
+*.tmp.py
+tmp

README.md

@@ -1,33 +1,17 @@
 ---
-title: Gpt2 Persian
-emoji: 🔥
-colorFrom: blue
-colorTo: green
+title: GPT2 Persian
+emoji: 🤘
+colorFrom: orange
+colorTo: red
 sdk: streamlit
 app_file: app.py
 pinned: false
 ---
 
-# Configuration
+# Streamlit demo for GPT2 Persian
 
-`title`: _string_  
-Display title for the Space
+### Launch demo:
 
-`emoji`: _string_  
-Space emoji (emoji-only character allowed)
-
-`colorFrom`: _string_  
-Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
-
-`colorTo`: _string_  
-Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
-
-`sdk`: _string_  
-Can be either `gradio` or `streamlit`
-
-`app_file`: _string_  
-Path to your main application file (which contains either `gradio` or `streamlit` Python code).  
-Path is relative to the root of the repository.
-
-`pinned`: _boolean_  
-Whether the Space stays on top of your list.
+```
+streamlit run app.py
+```

app.py

@@ -0,0 +1,158 @@
+import streamlit as st
+from transformers import pipeline, set_seed
+from transformers import AutoTokenizer
+from mtranslate import translate
+import random
+
+import meta
+import examples
+from normalizer import normalize
+from utils import (
+    remote_css,
+    local_css
+)
+
+
+class TextGeneration:
+    def __init__(self):
+        self.debug = False
+        self.dummy_output = "ناف جایی قرار گرفته که در واقع بندناف در داخل رحم در آنجا به شکم جنین وصل بوده‌است. " \
+                            "بندناف که جفت را به جنین متصل کرده بعد از تولد از نوزاد جدا می‌شود. برای جدا کردن بند ناف از دو پنس استفاده می‌کنند و بین آن دو را میبرند. پنس دیگری نزدیک شکم نوزاد قرار داده می‌شود که بعد از دو روز برداشته خواهد شد. بندناف باقی‌مانده طی ۱۵ روز خشک شده و می‌افتد و به جای آن اسکاری طبیعی به جای میماند. البته بر خلاف تصور عامه مردم شکل ناف در اثر بریدن بند ناف به وجود نمی‌آید و پیش از این در شکم مادر حالت ناف شکل گرفته‌است. شکل ناف در میان مردم مختلف متفاوت است و اندازه آن بین ۱.۵ تا ۲ سانتی‌متر است. تمام پستانداران جفت‌زیست ناف دارند. ناف در انسان‌ها به سادگی قابل مشاهده‌است."
+        self.tokenizer = None
+        self.generator = None
+        self.task = "text-generation"
+        self.model_name_or_path = "flax-community/gpt2-medium-persian"
+        set_seed(42)
+
+    def load(self):
+        if not self.debug:
+            self.tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path)
+            self.generator = pipeline(self.task, model=self.model_name_or_path, tokenizer=self.model_name_or_path)
+
+    def generate(self, prompt, **generation_kwargs):
+        if not self.debug:
+            generation_kwargs["num_return_sequences"] = 1
+
+            max_length = len(self.tokenizer(prompt)["input_ids"][0]) + generation_kwargs["max_length"]
+            generation_kwargs["max_length"] = max_length
+
+            return self.generator(
+                prompt,
+                **generation_kwargs,
+            )[0]["generated_text"]
+
+        return self.dummy_output
+
+
+@st.cache(allow_output_mutation=True)
+def load_text_generator():
+    generator = TextGeneration()
+    generator.load()
+    return generator
+
+
+def main():
+    st.set_page_config(
+        page_title="X",
+        page_icon="🧊",
+        layout="wide",
+        initial_sidebar_state="expanded"
+    )
+    remote_css("https://cdn.jsdelivr.net/gh/rastikerdar/vazir-font/dist/font-face.css")
+    local_css("assets/rtl.css")
+    generator = load_text_generator()
+
+    st.sidebar.markdown(meta.SIDEBAR_INFO)
+
+    max_length = st.sidebar.slider(
+        label='Max Length',
+        help="The maximum length of the sequence to be generated.",
+        min_value=1,
+        max_value=128,
+        value=50,
+        step=1
+    )
+    top_k = st.sidebar.slider(
+        label='Top-k',
+        help="The number of highest probability vocabulary tokens to keep for top-k-filtering",
+        min_value=40,
+        max_value=80,
+        value=50,
+        step=1
+    )
+    top_p = st.sidebar.slider(
+        label='Top-p',
+        help="Only the most probable tokens with probabilities that add up to `top_p` or higher are kept for "
+             "generation.",
+        min_value=0.0,
+        max_value=1.0,
+        value=0.95,
+        step=0.01
+    )
+    temperature = st.sidebar.slider(
+        label='Temperature',
+        help="The value used to module the next token probabilities",
+        min_value=0.1,
+        max_value=10.0,
+        value=1.0,
+        step=0.05
+    )
+    do_sample = st.sidebar.selectbox(
+        label='Sampling ?',
+        options=(True, False),
+        help="Whether or not to use sampling; use greedy decoding otherwise.",
+    )
+    translated = st.sidebar.selectbox(
+        label='Translation ?',
+        options=(True, False),
+        help="Will translate the result in English",
+    )
+    generation_kwargs = {
+        "max_length": max_length,
+        "top_k": top_k,
+        "top_p": top_p,
+        "temperature": temperature,
+        "do_sample": do_sample,
+    }
+
+    st.markdown(meta.HEADER_INFO)
+    prompts = list(examples.EXAMPLES.keys()) + ["Custom"]
+    prompt = st.selectbox('Examples', prompts, index=len(prompts) - 1)
+
+    if prompt == "Custom":
+        prompt_box = meta.PROMPT_BOX
+    else:
+        prompt_box = random.choice(examples.EXAMPLES[prompt])
+
+    text = st.text_area("Enter text", prompt_box)
+    generation_kwargs_ph = st.empty()
+
+    if st.button("Generate !"):
+        with st.spinner(text="Generating ..."):
+            generation_kwargs_ph.markdown(", ".join([f"`{k}`: {v}" for k, v in generation_kwargs.items()]))
+            text = normalize(text)
+            if text:
+                generated_text = generator.generate(text)
+                st.markdown(
+                    f'<p class="rtl rtl-box">'
+                    f'<span class="result-text">{text} <span>'
+                    f'<span class="result-text generated-text">{generated_text}</span>'
+                    f'</p>',
+                    unsafe_allow_html=True
+                )
+
+                if translated:
+                    translated_text = translate(text, "en", "fa")
+                    translated_generated_text = translate(generated_text, "en", "fa")
+
+                    st.markdown(
+                        f'<p class="ltr ltr-box">'
+                        f'<span class="result-text">{translated_text} <span>'
+                        f'<span class="result-text generated-text">{translated_generated_text}</span>'
+                        f'</p>',
+                        unsafe_allow_html=True
+                    )
+
+
+if __name__ == '__main__':
+    main()

assets/rtl.css

@@ -0,0 +1,22 @@
+.rtl,
+textarea {
+    font-family: Vazir !important;
+    text-align: right;
+    direction: rtl !important;
+}
+.rtl-box {
+    border-bottom: 1px solid #ddd;
+    padding-bottom: 20px;
+}
+.ltr {
+    text-align: left;
+    direction: ltr !important;
+}
+
+span.result-text {
+    padding: 3px 3px;
+    line-height: 32px;
+}
+span.generated-text {
+    background-color: rgb(118 200 147 / 13%);
+}

dictionary.py

@@ -0,0 +1,139 @@
+characters = {
+    "ك": "ک",
+    "دِ": "د",
+    "بِ": "ب",
+    "زِ": "ز",
+    "ذِ": "ذ",
+    "شِ": "ش",
+    "سِ": "س",
+    "ى": "ی",
+    "ي": "ی",
+    "ؤ": "و",
+    "ے": "ی",
+    "ۀ": "ه",
+    "ﭘ": "پ",
+    "ﮐ": "ک",
+    "ﯽ": "ی",
+    "ﺎ": "ا",
+    "ﺑ": "ب",
+    "ﺘ": "ت",
+    "ﺧ": "خ",
+    "ﺩ": "د",
+    "ﺱ": "س",
+    "ﻀ": "ض",
+    "ﻌ": "ع",
+    "ﻟ": "ل",
+    "ﻡ": "م",
+    "ﻢ": "م",
+    "ﻪ": "ه",
+    "ﻮ": "و",
+    "ﺍ": "ا",
+    "ة": "ه",
+    "ﯾ": "ی",
+    "ﯿ": "ی",
+    "ﺒ": "ب",
+    "ﺖ": "ت",
+    "ﺪ": "د",
+    "ﺮ": "ر",
+    "ﺴ": "س",
+    "ﺷ": "ش",
+    "ﺸ": "ش",
+    "ﻋ": "ع",
+    "ﻤ": "م",
+    "ﻥ": "ن",
+    "ﻧ": "ن",
+    "ﻭ": "و",
+    "ﺭ": "ر",
+    "ﮔ": "گ",
+    "إ": "ا",
+    "ٕ": " ",
+    "ھ": "ه",
+    "...": ".",
+    "…": ".",
+    "-": " - ",
+    "هٔ": "ه",
+    "ﻯ": "ی",
+    "ﻛ": "ک",
+    "ﭼ": "چ",
+    "ﺓ": "ه",
+    "ﻴ": "ی",
+    "ﻊ": "ع",
+    "ﮬ": "ه",
+    "ﺟ": "ج",
+    "ﺳ": "س",
+    "ﻦ": "ن",
+    "ﺬ": "ذ",
+    "ﺋ": "ئ",
+    "ﷲ": "لله",
+    "ﺞ": "ج",
+    "ﺙ": "ث",
+    "ﻗ": "ق",
+    "ﮪ": "ه",
+    "ﺰ": "ز",
+    "ﯼ": "ی",
+    "ٺ": "ت",
+    "ﺻ": "ص",
+    "ﻂ": "ط",
+    "ﻣ": "م",
+    "ﻈ": "ظ",
+    "ﺐ": "ب",
+    "ﻍ": "غ",
+    "ݸ": "و",
+    "ﻨ": "ن",
+    "ﻝ": "ل",
+    "ﻩ": "ه",
+    "ﻲ": "ی",
+    "ﻐ": "غ",
+    "ﺲ": "س",
+    "ﺁ": "آ",
+    "ڔ": "ر",
+    "ﺫ": "ذ",
+    "ﭻ": "چ",
+    "ﺠ": "ج",
+    "ﯙ": "و",
+    "ﮏ": "ک",
+    "ﺣ": "ح",
+    "ﺝ": "ج",
+    "ﺼ": "ص",
+    "ﻳ": "ی",
+    "ﻘ": "ق",
+    "ﺨ": "خ",
+    "ﻔ": "ف",
+    "ﻎ": "غ",
+    "ئ": "ی",
+    "ﻓ": "ف",
+    "ﻕ": "ق",
+    "ﮋ": "ژ",
+    "ﺗ": "ت",
+    "ﻁ": "ط",
+    "ﺯ": "ز",
+    "ﮕ": "گ",
+    "ﺌ": "ئ",
+    "ﺵ": "ش",
+    "ۮ": "د",
+    "ﻫ": "ه",
+    "ﻬ": "ه",
+    "ﻏ": "غ",
+    "ﻰ": "ی",
+    "﷼": "ریال",
+    "ﺿ": "ض",
+    "ﺛ": "ث",
+    "ݐ": "پ",
+    "ﺏ": "ب",
+    "ﭙ": "پ",
+    "ﭽ": "چ",
+    "ﺜ": "ث",
+    "ﻃ": "ط",
+    "ۂ": "ه",
+    "ﻑ": "ف",
+    "ﺕ": "ت",
+    "ﻞ": "ل",
+}
+
+special_tokens = {}
+
+words_map = {
+    "Leave a comment": "",
+    "[…]": "",
+    "[.]": "",
+}

examples.py

@@ -0,0 +1,7 @@
+EXAMPLES = {
+    "Scientific ...": [
+        "در یک اتفاق شگفت انگیز، پژوهشگران",
+        "عینک‌های هوشمند از جمله گجت‌هایی هستند که می‌توانند",
+        "وقتی ستاره‌ای بسیار بزرگ به پایان عمر خود می‌رسد، منفجر می‌شود و",
+    ],
+}

meta.py

@@ -0,0 +1,11 @@
+HEADER_INFO = """
+# GPT2 - Persian
+
+Persian GPT-2 demo. Part of the [Huggingface JAX/Flax event](
+https://discuss.huggingface.co/t/pretrain-gpt2-from-scratch-in-persian/7560).
+
+""".strip()
+SIDEBAR_INFO = """
+# Configuration
+""".strip()
+PROMPT_BOX = "متن خود را وارد کنید ..."

normalizer.py

@@ -0,0 +1,81 @@
+import hazm
+import re
+
+from regexes.currency import CURRENCY_REGEX
+from regexes.email import EMAIL_REGEX
+from regexes.latin import LATIN_REGEX
+from regexes.number import NUMBERS_REGEX
+from regexes.phone import PHONE_REGEX
+from regexes.quote import DOUBLE_QUOTE_REGEX, SINGLE_QUOTE_REGEX
+from regexes.url import URL_REGEX
+from regexes.persian import PERSIAN_REGEX
+import dictionary
+
+
+def make_trans(list_a, list_b):
+    return dict((ord(a), b) for a, b in zip(list_a, list_b))
+
+
+def multiple_replace(text, chars_to_mapping):
+    pattern = "|".join(map(re.escape, chars_to_mapping.keys()))
+    return re.sub(pattern, lambda m: chars_to_mapping[m.group()], str(text))
+
+
+ar2fa_digits = make_trans("٠١٢٣٤٥٦٧٨٩٪", "۰۱۲۳۴۵۶۷۸۹٪")
+fa2en_digits = make_trans("۰۱۲۳۴۵۶۷۸۹٪", "0123456789%")
+normalizer = hazm.Normalizer(persian_numbers=True, punctuation_spacing=False)
+
+
+def normalize(text, zwnj="\u200c", tokenized=False):
+    text = text.replace("\n", " ").replace("\t", " ")
+    text = re.sub(r"\u200c+", "\u200c", text)
+    text = text.replace('ـ', '')
+    text = normalizer.normalize(text)
+
+    if len(dictionary.characters) > 0:
+        text = multiple_replace(text, dictionary.characters)
+
+    if len(dictionary.words_map) > 0:
+        text = multiple_replace(text, dictionary.words_map)
+
+    text = text.translate(ar2fa_digits)
+    text = text.translate(fa2en_digits)
+
+    text = SINGLE_QUOTE_REGEX.sub("'", text)
+    text = DOUBLE_QUOTE_REGEX.sub('"', text)
+    text = CURRENCY_REGEX.sub(r" \1 ", text)
+    text = URL_REGEX.sub(" ", text)
+    text = EMAIL_REGEX.sub(" ", text)
+    text = PHONE_REGEX.sub(r" \1 ", text)
+    text = NUMBERS_REGEX.sub(r" \1 ", text)
+    text = LATIN_REGEX.sub(r" \1 ", text)
+
+    # Allow only english and persian characters
+    text = re.sub(PERSIAN_REGEX, " ", text)
+
+    text = text.replace(f" {zwnj} ", f"{zwnj}")
+    text = text.replace(f"{zwnj} ", f"{zwnj}")
+    text = text.replace(f" {zwnj}", f"{zwnj}")
+
+    if len(dictionary.special_tokens) > 0:
+        text = multiple_replace(text, dictionary.special_tokens)
+
+    tokens = []
+    for token in text.split():
+        token = token.strip()
+        if token:
+            if token.startswith(zwnj) and token.endswith(zwnj):
+                token = token[1:-1]
+            if token.startswith(zwnj):
+                token = token[1:]
+            elif token.endswith(zwnj):
+                token = token[:-1]
+            else:
+                token = token
+
+            tokens.append(token)
+
+    if tokenized:
+        return tokens
+
+    return " ".join(tokens)

regexes/currency.py

@@ -0,0 +1,23 @@
+import re
+
+CURRENCIES = {
+    "$": "USD",
+    "zł": "PLN",
+    "£": "GBP",
+    "¥": "JPY",
+    "฿": "THB",
+    "₡": "CRC",
+    "₦": "NGN",
+    "₩": "KRW",
+    "₪": "ILS",
+    "₫": "VND",
+    "€": "EUR",
+    "₱": "PHP",
+    "₲": "PYG",
+    "₴": "UAH",
+    "₹": "INR",
+    "﷼": "IRR",
+}
+CURRENCY_REGEX = re.compile(
+    "({})+".format("|".join(re.escape(c) for c in CURRENCIES.keys()))
+)

regexes/email.py

@@ -0,0 +1,6 @@
+import re
+
+EMAIL_REGEX = re.compile(
+    r"(?:^|(?<=[^\w@.)]))([\w+-](\.(?!\.))?)*?[\w+-](@|[(<{\[]at[)>}\]])(?:(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)(?:\.(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)*(?:\.(?:[a-z\\u00a1-\\uffff]{2,}))",
+    flags=re.IGNORECASE | re.UNICODE,
+)

regexes/latin.py

@@ -0,0 +1,13 @@
+import re
+
+LATIN_WITH_SPECIAL_REGEX = re.compile(
+    r"(\b(?!URL|EMAIL|PHONE|NUMBER|CUR|LATIN\b)[0-9a-zA-Z]+)"
+)
+
+LATIN_REGEX = re.compile(
+    r"([0-9a-zA-Z]+)"
+)
+
+LATIN_SPACES_REGEX = re.compile(
+    r"([0-9a-zA-Z])"
+)

regexes/number.py

@@ -0,0 +1,5 @@
+import re
+
+NUMBERS_REGEX = re.compile(
+    r"(?:^|(?<=[^\w,.]))[+–-]?(([1-9]\d{0,2}(,\d{3})+(\.\d*)?)|([1-9]\d{0,2}([ .]\d{3})+(,\d*)?)|(\d*?[.,]\d+)|\d+)(?:$|(?=\b))"
+)

regexes/persian.py

@@ -0,0 +1,19 @@
+import re
+
+
+PERSIAN_ALPHA = "ءآئابتثجحخدذرزسشصضطظعغفقلمنهوپچژکگیە"  # noqa: E501
+PERSIAN_DIGIT = "۰۱۲۳۴۵۶۷۸۹"
+
+
+ZWNJ = "\u200c"
+PUNK = '\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\_\`\{\|\}\~\«\»\؟\:\×\٬\٫\﷼\٪\،'
+
+PERSIAN = (
+    "a-zA-Z0-9" +
+    PERSIAN_ALPHA +
+    PERSIAN_DIGIT +
+    ZWNJ +
+    PUNK
+)
+
+PERSIAN_REGEX = r"[^" + PERSIAN + "+]"

regexes/phone.py

@@ -0,0 +1,6 @@
+import re
+
+
+PHONE_REGEX = re.compile(
+    r"((?:^|(?<=[^\w)]))(((\+?[01])|(\+\d{2}))[ .-]?)?(\(?\d{3,4}\)?/?[ .-]?)?(\d{3}[ .-]?\d{4})(\s?(?:ext\.?|[#x-])\s?\d{2,6})?(?:$|(?=\W)))|\+?\d{4,5}[ .-/]\d{6,9}"
+)

regexes/punk.py

@@ -0,0 +1,5 @@
+import re
+
+PUNK_REGEX = re.compile(
+    r"([\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\=\?\@\[\\\]\^\_\`\{\|\}\~\«\»\⸮\؟\،\٬\٫\؛])"
+)

regexes/quote.py

@@ -0,0 +1,25 @@
+import re
+
+
+strange_double_quotes = [
+    "«",
+    "‹",
+    "»",
+    "›",
+    "„",
+    "“",
+    "‟",
+    "”",
+    "❝",
+    "❞",
+    "❮",
+    "❯",
+    "〝",
+    "〞",
+    "〟",
+    "＂",
+]
+strange_single_quotes = ["‘", "‛", "’", "❛", "❜", "`", "´", "‘", "’"]
+
+DOUBLE_QUOTE_REGEX = re.compile("|".join(strange_double_quotes))
+SINGLE_QUOTE_REGEX = re.compile("|".join(strange_single_quotes))

regexes/url.py

@@ -0,0 +1,38 @@
+import re
+
+URL_REGEX = re.compile(
+    r"(?:^|(?<![\w\/\.]))"
+    # protocol identifier
+    # r"(?:(?:https?|ftp)://)"  <-- alt?
+    r"(?:(?:https?:\/\/|ftp:\/\/|www\d{0,3}\.))"
+    # user:pass authentication
+    r"(?:\S+(?::\S*)?@)?" r"(?:"
+    # IP address exclusion
+    # private & local networks
+    r"(?!(?:10|127)(?:\.\d{1,3}){3})"
+    r"(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})"
+    r"(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})"
+    # IP address dotted notation octets
+    # excludes loopback network 0.0.0.0
+    # excludes reserved space >= 224.0.0.0
+    # excludes network & broadcast addresses
+    # (first & last IP address of each class)
+    r"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])"
+    r"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}"
+    r"(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))"
+    r"|"
+    # host name
+    r"(?:(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)"
+    # domain name
+    r"(?:\.(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)*"
+    # TLD identifier
+    r"(?:\.(?:[a-z\\u00a1-\\uffff]{2,}))" r"|" r"(?:(localhost))" r")"
+    # port number
+    r"(?::\d{2,5})?"
+    # resource path
+    r"(?:\/[^\)\]\}\s]*)?",
+    # r"(?:$|(?![\w?!+&\/\)]))",
+    # @jfilter: I removed the line above from the regex because I don't understand what it is used for, maybe it was useful?
+    # But I made sure that it does not include ), ] and } in the URL.
+    flags=re.UNICODE | re.IGNORECASE,
+)

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+hazm
+Pillow
+mtranslate
+torch
+transformers

utils.py

@@ -0,0 +1,35 @@
+import streamlit as st
+import json
+from PIL import Image
+
+
+def load_image(image_path, image_resize=None):
+    image = Image.open(image_path)
+    if isinstance(image_resize, tuple):
+        image.resize(image_resize)
+    return image
+
+
+def load_text(text_path):
+    text = ''
+    with open(text_path) as f:
+        text = f.read()
+
+    return text
+
+
+def load_json(json_path):
+    jdata = ''
+    with open(json_path) as f:
+        jdata = json.load(f)
+
+    return jdata
+
+
+def local_css(css_path):
+    with open(css_path) as f:
+        st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)
+
+
+def remote_css(css_url):
+    st.markdown(f'<link href="{css_url}" rel="stylesheet">', unsafe_allow_html=True)