Document RNN (#4)

.github/workflows/ci-cd.yml

@@ -31,7 +31,6 @@ jobs:
         with:
           python-version: "3.10"
       - name: Package up SDK
-        if: github.ref == 'refs/heads/master'
         run: python setup.py sdist
       - name: Publish a Python distribution to PyPI
         if: github.ref == 'refs/heads/master'

.gitignore

@@ -1,2 +1,3 @@
 .idea/
 .DS_Store
+__pycache__

docs/source/img/rnn-4-black-boxes-connected.drawio

@@ -0,0 +1,121 @@
+<mxfile host="app.diagrams.net" modified="2024-03-19T01:01:04.926Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" etag="OgWHmKqu6mVN4yDKoCwM" version="24.0.7" type="device">
+  <diagram name="Page-1" id="gxr7cFC-hZQY0lpAcxoR">
+    <mxGraphModel dx="816" dy="516" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-21" value="&lt;font size=&quot;1&quot; data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;&quot; color=&quot;#ffffff&quot;&gt;&lt;b style=&quot;font-size: 27px;&quot;&gt;f&lt;/b&gt;&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" vertex="1" parent="1">
+          <mxGeometry x="40" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-22" value="&lt;font color=&quot;#ffffff&quot; face=&quot;Italianno&quot; style=&quot;font-size: 27px;&quot;&gt;f&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" vertex="1" parent="1">
+          <mxGeometry x="180" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-23" value="&lt;span style=&quot;color: rgb(255, 255, 255); font-family: Italianno; font-size: 27px;&quot;&gt;f&lt;/span&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" vertex="1" parent="1">
+          <mxGeometry x="320" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-24" value="&lt;span style=&quot;color: rgb(255, 255, 255); font-family: Italianno; font-size: 27px;&quot;&gt;f&lt;/span&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" vertex="1" parent="1">
+          <mxGeometry x="460" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-25" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="69.5" y="439" as="sourcePoint" />
+            <mxPoint x="69.5" y="389" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-26" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;h&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="45" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-27" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="69.5" y="310" as="sourcePoint" />
+            <mxPoint x="69.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-28" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;e&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="45" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-29" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="185" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-30" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="325" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-31" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;o&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="465" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-32" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="209.5" y="310" as="sourcePoint" />
+            <mxPoint x="209.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-33" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="349.5" y="310" as="sourcePoint" />
+            <mxPoint x="349.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-34" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="489.5" y="310" as="sourcePoint" />
+            <mxPoint x="489.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-35" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;e&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="185" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-36" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="325" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-37" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="465" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-38" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="209.5" y="440" as="sourcePoint" />
+            <mxPoint x="209.5" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-39" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="349.5" y="440" as="sourcePoint" />
+            <mxPoint x="349.5" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-40" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="489.5" y="440" as="sourcePoint" />
+            <mxPoint x="489.5" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-44" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="110" y="349.78" as="sourcePoint" />
+            <mxPoint x="170.5" y="349.78" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-45" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="250" y="349.76" as="sourcePoint" />
+            <mxPoint x="310.5" y="349.76" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-46" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="394.75" y="349.76" as="sourcePoint" />
+            <mxPoint x="455.25" y="349.76" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-47" value="&lt;font data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;sub style=&quot;font-size: 25px;&quot;&gt;1&lt;/sub&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=25;fontStyle=1" vertex="1" parent="1">
+          <mxGeometry x="110" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-48" value="&lt;font style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;/span&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;sub style=&quot;font-size: 25px;&quot;&gt;2&lt;/sub&gt;&lt;/span&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Italianno;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DItalianno;fontSize=25;fontStyle=1" vertex="1" parent="1">
+          <mxGeometry x="250" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="o5WZRm4PuDRFcwwRBSCM-49" value="&lt;font style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;/span&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;sub style=&quot;font-size: 25px;&quot;&gt;3&lt;/sub&gt;&lt;/span&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Italianno;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DItalianno;fontSize=25;fontStyle=1" vertex="1" parent="1">
+          <mxGeometry x="390" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

docs/source/img/rnn-4-black-boxes.drawio

@@ -0,0 +1,94 @@
+<mxfile host="app.diagrams.net" modified="2024-03-19T00:55:52.180Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" etag="0mAcZEyuQtVV9Bg2w-Tf" version="24.0.7" type="device">
+  <diagram name="Page-1" id="DUD_6-T85kScICrpKMMz">
+    <mxGraphModel dx="1536" dy="972" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="LGyqkTtVOXbUGTZdzTyq-1" value="&lt;font size=&quot;1&quot; data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;&quot; color=&quot;#ffffff&quot;&gt;&lt;b style=&quot;font-size: 27px;&quot;&gt;f&lt;/b&gt;&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="20" y="300" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="LGyqkTtVOXbUGTZdzTyq-2" value="&lt;font color=&quot;#ffffff&quot; face=&quot;Italianno&quot; style=&quot;font-size: 27px;&quot;&gt;f&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="160" y="300" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="LGyqkTtVOXbUGTZdzTyq-4" value="&lt;span style=&quot;color: rgb(255, 255, 255); font-family: Italianno; font-size: 27px;&quot;&gt;f&lt;/span&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="300" y="300" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="LGyqkTtVOXbUGTZdzTyq-5" value="&lt;span style=&quot;color: rgb(255, 255, 255); font-family: Italianno; font-size: 27px;&quot;&gt;f&lt;/span&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="440" y="300" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="LGyqkTtVOXbUGTZdzTyq-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="49.5" y="419" as="sourcePoint" />
+            <mxPoint x="49.5" y="369" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-1" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;h&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="25" y="430" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-2" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="49.5" y="290" as="sourcePoint" />
+            <mxPoint x="49.5" y="240" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-3" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;e&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="25" y="180" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-4" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="165" y="180" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-5" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="305" y="180" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-6" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;o&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="445" y="180" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="189.5" y="290" as="sourcePoint" />
+            <mxPoint x="189.5" y="240" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-9" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="329.5" y="290" as="sourcePoint" />
+            <mxPoint x="329.5" y="240" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="469.5" y="290" as="sourcePoint" />
+            <mxPoint x="469.5" y="240" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-11" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;e&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="165" y="430" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-12" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="305" y="430" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-13" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="445" y="430" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="189.5" y="420" as="sourcePoint" />
+            <mxPoint x="189.5" y="370" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="329.5" y="420" as="sourcePoint" />
+            <mxPoint x="329.5" y="370" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="hK792VXiPIr8ubialXFB-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" edge="1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="469.5" y="420" as="sourcePoint" />
+            <mxPoint x="469.5" y="370" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

docs/source/img/rnn-multi-sequences.drawio

@@ -0,0 +1,250 @@
+<mxfile host="app.diagrams.net" modified="2024-03-19T01:28:19.045Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" etag="1-yK62UkPGlKoTsEWqch" version="24.0.7" type="device">
+  <diagram name="Page-1" id="6HRoGfWBaaDKhnXAU6vd">
+    <mxGraphModel dx="2156" dy="1926" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="vckTE8xcX2gjwNGocpAb-1" value="&lt;font size=&quot;1&quot; data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;&quot; color=&quot;#ffffff&quot;&gt;&lt;b style=&quot;font-size: 27px;&quot;&gt;f&lt;/b&gt;&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="40" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-2" value="&lt;font color=&quot;#ffffff&quot; face=&quot;Italianno&quot; style=&quot;font-size: 27px;&quot;&gt;f&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="180" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-3" value="&lt;span style=&quot;color: rgb(255, 255, 255); font-family: Italianno; font-size: 27px;&quot;&gt;f&lt;/span&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="320" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-4" value="&lt;span style=&quot;color: rgb(255, 255, 255); font-family: Italianno; font-size: 27px;&quot;&gt;f&lt;/span&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="460" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-5" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="69.5" y="439" as="sourcePoint" />
+            <mxPoint x="69.5" y="389" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-6" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;h&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="45" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-7" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="69.5" y="310" as="sourcePoint" />
+            <mxPoint x="69.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-8" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;e&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="45" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-9" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="185" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-10" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="325" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-11" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;o&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="465" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-12" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="209.5" y="310" as="sourcePoint" />
+            <mxPoint x="209.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="349.5" y="310" as="sourcePoint" />
+            <mxPoint x="349.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="489.5" y="310" as="sourcePoint" />
+            <mxPoint x="489.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-15" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;e&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="185" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-16" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="325" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-17" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;l&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="465" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-18" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="209.5" y="440" as="sourcePoint" />
+            <mxPoint x="209.5" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-19" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="349.5" y="440" as="sourcePoint" />
+            <mxPoint x="349.5" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-20" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="489.5" y="440" as="sourcePoint" />
+            <mxPoint x="489.5" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-21" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="110" y="349.78" as="sourcePoint" />
+            <mxPoint x="170.5" y="349.78" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-22" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="250" y="349.76" as="sourcePoint" />
+            <mxPoint x="310.5" y="349.76" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="394.75" y="349.76" as="sourcePoint" />
+            <mxPoint x="455.25" y="349.76" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-24" value="&lt;font data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;sub style=&quot;font-size: 25px;&quot;&gt;1&lt;/sub&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=25;fontStyle=1" parent="1" vertex="1">
+          <mxGeometry x="110" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-25" value="&lt;font style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;/span&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;sub style=&quot;font-size: 25px;&quot;&gt;2&lt;/sub&gt;&lt;/span&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Italianno;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DItalianno;fontSize=25;fontStyle=1" parent="1" vertex="1">
+          <mxGeometry x="250" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-26" value="&lt;font style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;/span&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;sub style=&quot;font-size: 25px;&quot;&gt;3&lt;/sub&gt;&lt;/span&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Italianno;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DItalianno;fontSize=25;fontStyle=1" parent="1" vertex="1">
+          <mxGeometry x="390" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-27" value="&lt;font face=&quot;Ubuntu&quot; size=&quot;1&quot; style=&quot;&quot; color=&quot;#ffffff&quot;&gt;&lt;b style=&quot;font-size: 23px;&quot;&gt;RNN&lt;/b&gt;&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="-140" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-28" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;hell&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="-135" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-29" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;ello&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="-135" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-30" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="-110.25999999999999" y="310" as="sourcePoint" />
+            <mxPoint x="-110.25999999999999" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-31" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="-110.25999999999999" y="440" as="sourcePoint" />
+            <mxPoint x="-110.25999999999999" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-33" value="" style="shape=flexArrow;endArrow=classic;html=1;rounded=0;strokeWidth=3;" parent="1" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="-70" y="350" as="sourcePoint" />
+            <mxPoint x="30" y="350" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-34" value="&lt;b&gt;&lt;font style=&quot;font-size: 19px;&quot; face=&quot;Ubuntu&quot;&gt;Unfold&lt;/font&gt;&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="-50" y="290" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-35" value="&lt;font size=&quot;1&quot; data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;&quot; color=&quot;#ffffff&quot;&gt;&lt;b style=&quot;font-size: 27px;&quot;&gt;f&lt;/b&gt;&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="40" y="-50" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-36" value="&lt;font color=&quot;#ffffff&quot; face=&quot;Italianno&quot; style=&quot;font-size: 27px;&quot;&gt;f&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="180" y="-50" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-37" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="69.5" y="69" as="sourcePoint" />
+            <mxPoint x="69.5" y="19" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-38" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;c&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="45" y="80" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-39" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="69.5" y="-60" as="sourcePoint" />
+            <mxPoint x="69.5" y="-110" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-40" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;a&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="45" y="-170" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-41" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;t&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="185" y="-170" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-42" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="209.5" y="-60" as="sourcePoint" />
+            <mxPoint x="209.5" y="-110" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-43" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;a&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="185" y="80" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-44" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="209.5" y="70" as="sourcePoint" />
+            <mxPoint x="209.5" y="20" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-45" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="110" y="-20.220000000000027" as="sourcePoint" />
+            <mxPoint x="170.5" y="-20.220000000000027" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-46" value="&lt;font data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;sub style=&quot;font-size: 25px;&quot;&gt;1&lt;/sub&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=25;fontStyle=1" parent="1" vertex="1">
+          <mxGeometry x="110" y="-50" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-47" value="&lt;font face=&quot;Ubuntu&quot; size=&quot;1&quot; style=&quot;&quot; color=&quot;#ffffff&quot;&gt;&lt;b style=&quot;font-size: 23px;&quot;&gt;RNN&lt;/b&gt;&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="-140" y="-50" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-48" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;ca&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="-135" y="80" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-49" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;at&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="-135" y="-170" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-50" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="-110.25999999999999" y="-60" as="sourcePoint" />
+            <mxPoint x="-110.25999999999999" y="-110" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-51" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="-110.25999999999999" y="70" as="sourcePoint" />
+            <mxPoint x="-110.25999999999999" y="20" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-53" value="" style="shape=flexArrow;endArrow=classic;html=1;rounded=0;strokeWidth=3;" parent="1" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="-70" y="-20" as="sourcePoint" />
+            <mxPoint x="30" y="-20" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-54" value="&lt;b&gt;&lt;font style=&quot;font-size: 19px;&quot; face=&quot;Ubuntu&quot;&gt;Unfold&lt;/font&gt;&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="-50" y="-80" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-55" value="" style="endArrow=none;dashed=1;html=1;dashPattern=1 3;strokeWidth=3;rounded=0;" parent="1" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="-225" y="104.82000000000001" as="sourcePoint" />
+            <mxPoint x="-150" y="105.18" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-56" value="" style="endArrow=none;dashed=1;html=1;dashPattern=1 3;strokeWidth=3;rounded=0;" parent="1" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="-240" y="474.63" as="sourcePoint" />
+            <mxPoint x="-165" y="474.99" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-57" value="&lt;b&gt;&lt;font style=&quot;font-size: 19px;&quot; face=&quot;Ubuntu&quot;&gt;sequence 1&lt;/font&gt;&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="-345" y="90" width="110" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vckTE8xcX2gjwNGocpAb-58" value="&lt;b&gt;&lt;font style=&quot;font-size: 19px;&quot; face=&quot;Ubuntu&quot;&gt;sequence 2&lt;/font&gt;&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="-360" y="460" width="110" height="30" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

docs/source/img/rnn.drawio

@@ -0,0 +1,149 @@
+<mxfile host="app.diagrams.net" modified="2024-03-19T01:41:00.069Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" etag="gBXA4TJMJ-xFoxQL8in4" version="24.0.7" type="device">
+  <diagram name="Page-1" id="DUD_6-T85kScICrpKMMz">
+    <mxGraphModel dx="1783" dy="590" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-1" value="&lt;font size=&quot;1&quot; data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;&quot; color=&quot;#ffffff&quot;&gt;&lt;b style=&quot;font-size: 27px;&quot;&gt;f&lt;/b&gt;&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="40" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-2" value="&lt;font color=&quot;#ffffff&quot; face=&quot;Italianno&quot; style=&quot;font-size: 27px;&quot;&gt;f&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="180" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-4" value="&lt;span style=&quot;color: rgb(255, 255, 255); font-family: Italianno; font-size: 27px;&quot;&gt;f&lt;/span&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="460" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-5" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="69.5" y="439" as="sourcePoint" />
+            <mxPoint x="69.5" y="389" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-6" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;x&lt;sub&gt;1&lt;/sub&gt;&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="45" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-7" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="69.5" y="310" as="sourcePoint" />
+            <mxPoint x="69.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-8" value="&lt;font face=&quot;Ubuntu&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;o&lt;sub&gt;1&lt;/sub&gt;&lt;/b&gt;&lt;/span&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="45" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-9" value="&lt;b style=&quot;font-family: Ubuntu; font-size: 20px;&quot;&gt;o&lt;/b&gt;&lt;b style=&quot;font-family: Ubuntu; font-size: 16.6667px;&quot;&gt;&lt;sub&gt;2&lt;/sub&gt;&lt;/b&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="185" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-11" value="&lt;b style=&quot;font-family: Ubuntu; font-size: 20px;&quot;&gt;o&lt;/b&gt;&lt;b style=&quot;font-family: Ubuntu; font-size: 13.8889px;&quot;&gt;n&lt;/b&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="465" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-12" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="209.5" y="310" as="sourcePoint" />
+            <mxPoint x="209.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="489.5" y="310" as="sourcePoint" />
+            <mxPoint x="489.5" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-15" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;x&lt;sub&gt;2&lt;/sub&gt;&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="185" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-17" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;&quot;&gt;&lt;b style=&quot;&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;x&lt;/span&gt;&lt;span style=&quot;font-size: 16.6667px;&quot;&gt;&lt;sub&gt;n&lt;/sub&gt;&lt;/span&gt;&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="465" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-18" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="209.5" y="440" as="sourcePoint" />
+            <mxPoint x="209.5" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-20" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="489.5" y="440" as="sourcePoint" />
+            <mxPoint x="489.5" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-21" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="110" y="349.78" as="sourcePoint" />
+            <mxPoint x="170.5" y="349.78" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-22" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="250" y="349.76" as="sourcePoint" />
+            <mxPoint x="310.5" y="349.76" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#B266FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="394.75" y="349.76" as="sourcePoint" />
+            <mxPoint x="455.25" y="349.76" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-24" value="&lt;font data-font-src=&quot;https://fonts.googleapis.com/css?family=Italianno&quot; face=&quot;Italianno&quot; style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;sub style=&quot;font-size: 25px;&quot;&gt;1&lt;/sub&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=25;fontStyle=1" parent="1" vertex="1">
+          <mxGeometry x="110" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-25" value="&lt;font style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;h&lt;/span&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;sub style=&quot;font-size: 25px;&quot;&gt;2&lt;/sub&gt;&lt;/span&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Italianno;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DItalianno;fontSize=25;fontStyle=1" parent="1" vertex="1">
+          <mxGeometry x="250" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-26" value="&lt;font style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;&lt;span style=&quot;font-size: 25px;&quot;&gt;hn&lt;/span&gt;&lt;/span&gt;&lt;/font&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontFamily=Italianno;fontSource=https%3A%2F%2Ffonts.googleapis.com%2Fcss%3Ffamily%3DItalianno;fontSize=25;fontStyle=1" parent="1" vertex="1">
+          <mxGeometry x="390" y="320" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-27" value="&lt;font face=&quot;Ubuntu&quot; size=&quot;1&quot; style=&quot;&quot; color=&quot;#ffffff&quot;&gt;&lt;b style=&quot;font-size: 23px;&quot;&gt;RNN&lt;/b&gt;&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#000000;strokeColor=none;fontColor=#FF6666;" parent="1" vertex="1">
+          <mxGeometry x="-140" y="320" width="60" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-28" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;font-size: 20px;&quot;&gt;&lt;b&gt;seq&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="-135" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-29" value="&lt;font size=&quot;1&quot; face=&quot;Ubuntu&quot;&gt;&lt;b style=&quot;font-size: 13px;&quot;&gt;output&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" parent="1" vertex="1">
+          <mxGeometry x="-135" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-30" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="-110.25999999999999" y="310" as="sourcePoint" />
+            <mxPoint x="-110.25999999999999" y="260" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-31" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=3;strokeColor=#6666FF;fontColor=#FF6666;" parent="1" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="-110.25999999999999" y="440" as="sourcePoint" />
+            <mxPoint x="-110.25999999999999" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-32" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;entryX=0.25;entryY=1;entryDx=0;entryDy=0;strokeWidth=3;curved=1;" parent="1" source="Kn0003oJxsBQeWTrvDDb-27" target="Kn0003oJxsBQeWTrvDDb-27" edge="1">
+          <mxGeometry relative="1" as="geometry">
+            <Array as="points">
+              <mxPoint x="-125" y="290" />
+              <mxPoint x="-210" y="290" />
+              <mxPoint x="-210" y="410" />
+              <mxPoint x="-125" y="410" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-33" value="" style="shape=flexArrow;endArrow=classic;html=1;rounded=0;strokeWidth=3;" parent="1" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="-70" y="350" as="sourcePoint" />
+            <mxPoint x="30" y="350" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Kn0003oJxsBQeWTrvDDb-34" value="&lt;b&gt;&lt;font style=&quot;font-size: 19px;&quot; face=&quot;Ubuntu&quot;&gt;Unfold&lt;/font&gt;&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="-50" y="290" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="phTSMtF67GbWN2Al6Nvf-5" value="&lt;b&gt;&lt;font style=&quot;font-size: 25px;&quot; face=&quot;Ubuntu&quot;&gt;. . .&lt;/font&gt;&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="323" y="330" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="phTSMtF67GbWN2Al6Nvf-6" value="&lt;b style=&quot;font-family: Ubuntu; font-size: 20px;&quot;&gt;o&lt;/b&gt;&lt;b style=&quot;font-family: Ubuntu; font-size: 13.8889px;&quot;&gt;&lt;sub&gt;i&lt;/sub&gt;&lt;/b&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#FF3399;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="323" y="200" width="50" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="phTSMtF67GbWN2Al6Nvf-7" value="&lt;font face=&quot;Ubuntu&quot; style=&quot;&quot;&gt;&lt;b style=&quot;&quot;&gt;&lt;span style=&quot;font-size: 20px;&quot;&gt;x&lt;/span&gt;&lt;span style=&quot;font-size: 13.8889px;&quot;&gt;&lt;sub&gt;i&lt;/sub&gt;&lt;/span&gt;&lt;/b&gt;&lt;/font&gt;" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;fillColor=#00CC00;fontColor=#FFFFFF;" vertex="1" parent="1">
+          <mxGeometry x="323" y="450" width="50" height="50" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

docs/source/lamassu.rst

@@ -5,5 +5,5 @@ Lamassu
 .. toctree::
    :maxdepth: 100
 
-   rnn/vanilla
+   rnn/rnn
    speech/sampling.rst

docs/source/rnn/rnn.rst

@@ -0,0 +1,612 @@
+================================================
+Introduction to Recurrent Neural Networks (RNNs)
+================================================
+
+.. admonition:: Prerequisite
+
+    This article has the following prerequisites:
+
+    1. *Chapter 4 - Artificial Neural Networks* (p. 81) of `MACHINE LEARNING by Mitchell, Thom M. (1997)`_ Paperback
+    2. *Deep Learning (Adaptive Computation and Machine Learning series), Ian Goodfellow*
+
+.. contents:: Table of Contents
+    :depth: 2
+
+We all heard of this buz word "LLM" (Large Language Model). But let's put that aside for just a second and look at a
+much simpler one called "character-level language model" where, for example, we input a prefix of a word such as
+"hell" and the model outputs a complete word "hello". That is, this language model predicts the next character of a
+character sequence
+
+This is like a Math function where we have:
+
+.. math::
+
+    f(\text{“hell"}) = \text{“hello"}
+
+.. NOTE::
+
+    We call inputs like "hell" as **sequence**
+
+How do we obtain a function like this? One approach is to have 4 black boxes, each of which takes a single character as
+input and calculates an output:
+
+.. figure:: ../img/rnn-4-black-boxes.png
+    :align: center
+    :width: 50%
+
+But one might have noticed that if the 3rd function (box) produces :math:`f(‘l') = ‘l'`, then why would the 4th function
+(box), given the same input, gives a different output of 'o'? This suggest that we should take the "**history**" into
+account. Instead of having :math:`f` depend on 1 parameter, we now have it take 2 parameters.
+
+1: a character;
+2: a variable that summarizes the previous calculations:
+
+   .. figure:: ../img/rnn-4-black-boxes-connected.png
+       :align: center
+       :width: 50%
+
+Now it makes much more sense with:
+
+.. math::
+
+    f(\text{‘l'}, h_2) = \text{‘l'}
+
+    f(\text{‘l'}, h_3) = \text{‘o'}
+
+But what if we want to predict a longer or shorter word? For example, how about predicting "cat" by "ca"? That's simple,
+we will have 2 black boxes to do the work.
+
+.. figure:: ../img/rnn-multi-sequences.png
+    :align: center
+
+What if the function :math:`f` is not smart enough to produce the correct output everytime? We will simply collect a lot
+of examples such as "cat" and "hello", and feed them into the boxes to train them until they can output correct
+vocabulary like "cat" and "hello".
+
+This is the idea behind RNN
+
+- It's recurrent because the boxed function gets invoked repeatedly for each element of the sequence. In the case of our
+  character-level language model, element is a character such as "e" and sequence is a string like "hell"
+
+  .. figure:: ../img/rnn.png
+      :align: center
+
+Each function :math:`f` is a network unit containing 2 perceptrons. One perceptron computes the "history" like
+:math:`h_1`, :math:`h_2`, :math:`h_3`. Its formula is very similar to that of perceptron:
+
+.. math::
+
+    h^{(t)} = g_1\left( W_{hh}h^{(t - 1)} + W_{xh}x^{(t)} + b_h \right)
+
+where :math:`t` is the index of the "black boxes" shown above. In our example of "hell",
+:math:`t \in \{ 1, 2, 3, 4 \}`
+
+The other perceptron computes the output like 'e', 'l', 'l', 'o'. We call those value :math:`y` which is computed as
+
+.. math::
+
+    o^{(t)} = g_2\left( W_{yh}h^{(t)} + b_o \right)
+
+.. admonition:: What are :math:`g_1` and :math:`g_2`?
+
+    They are *activation functions* which are used to change the linear function in a perceptron to a non-linear
+    function. Please refer to `MACHINE LEARNING by Mitchell, Thom M. (1997)`_ Paperback (page 96) for why we bump it
+    to non-linear
+
+    A typical activation function for :math:`g_1` is :math:`tanh`:
+
+    .. math::
+
+        tanh(x) = \frac{e^x - e^{-x}}{e^x + e^{-x}}
+
+    In practice, :math:`g_2` is constance, i.e. :math:`g_2 = 1`
+
+
+Forward Propagation Equations for RNN
+-------------------------------------
+
+We now develop the forward propagation equations for the RNN. We assume the hyperbolic tangent activation function and
+that the output is discrete, as if the RNN is used to predict words or characters. A natural way to represent discrete
+variables is to regard the output :math:`\boldsymbol{o}` as giving the unnormalized log probabilities of each possible value of
+the discrete variable. We can then apply the softmax (we will disucss softmax function in the next section) operation as
+a post-processing step to obtain a vector :math:`\boldsymbol{\hat{y}}` of normalized probabilities over the output. Forward
+propagation begins with a specification of the initial state :math:`\boldsymbol{h}^{(0)}`. Then, for each time step from
+:math:`t = 1` to :math:`t = \tau`, we apply the following update equations:
+
+.. math::
+
+    \color{green} \boxed{
+        \begin{gather*}
+            \boldsymbol{h}^{(t)} = \tanh\left( \boldsymbol{W_{hh}}h^{(t - 1)} + \boldsymbol{W_{xh}}x^{(t)} + \boldsymbol{b_h} \right) \\ \\
+            \boldsymbol{o}^{(t)} = \boldsymbol{W_{yh}}\boldsymbol{h}^{(t)} + \boldsymbol{b_o} \\ \\
+            \boldsymbol{\hat{y}} = softmax(\boldsymbol{o}^{(t)})
+        \end{gather*}
+    }
+
+Note that this recurrent network maps an input sequence to an output sequence of the same length.
+
+Loss Function of RNN
+--------------------
+
+According to the discussion of `MACHINE LEARNING by Mitchell, Thom M. (1997)`_, the key for training RNN or any neural
+network is through "specifying a measure for the training error". We call this measure a *loss function*.
+
+In RNN, the total loss for a given sequence of input :math:`\boldsymbol{x}` paired with a sequence of expected
+:math:`\boldsymbol{y}` is the sum of the losses over all the time steps, i.e.
+
+.. math::
+
+    \mathcal{L}\left( \{ \boldsymbol{x}^{(1)}, ..., \boldsymbol{x}^{(\tau)} \}, \{ \boldsymbol{y}^{(1)}, ..., \boldsymbol{y}^{(\tau)} \} \right) = \sum_t^{\tau} \mathcal{L}^{(t)} = \sum_t^{\tau}\log\boldsymbol{\hat{y}}^{(t)}
+
+Why would we have :math:`\mathcal{L}^{(t)} = \log\boldsymbol{\hat{y}}^{(t)}`? We need to learn *Softmax Activation* first.
+
+.. admonition:: Softmax Function by `Wikipedia <https://en.wikipedia.org/wiki/Softmax_function>`_
+
+    The softmax function takes as input a vector :math:`z` of :math:`K` real numbers, and normalizes it into a
+    probability distribution consisting of :math:`K` probabilities proportional to the exponentials of the input
+    numbers. That is, prior to applying softmax, some vector components could be negative, or greater than one; and
+    might not sum to 1; but after applying softmax, each component will be in the interval :math:`(0, 1)` and the
+    components will add up to 1, so that they can be interpreted as probabilities. Furthermore, the larger input
+    components will correspond to larger probabilities.
+
+    For a vector :math:`z` of :math:`K` real numbers, the the standard (unit) softmax function
+    :math:`\sigma: \mathbb{R}^K \mapsto (0, 1)^K`, where :math:`K \ge 1` is defined by
+
+    .. math::
+
+        \sigma(\boldsymbol{z})_i = \frac{e^{z_i}}{\sum_{j = 1}^Ke^{z_j}}
+
+    where :math:`i = 1, 2, ..., K` and :math:`\boldsymbol{x} = (x_1, x_2, ..., x_K) \in \mathbb{R}^K`
+
+In the context of RNN,
+
+.. math::
+
+    \sigma(\boldsymbol{o})_i = -\frac{e^{o_i}}{\sum_{j = 1}^ne^{o_j}}
+
+where
+
+- :math:`n` is the length of a sequence feed into the RNN
+- :math:`o_i` is the output by perceptron unit `i`
+- :math:`i = 1, 2, ..., n`,
+- :math:`\boldsymbol{o} = (o_1, o_2, ..., o_n) \in \mathbb{R}^n`
+
+The softmax function takes an N-dimensional vector of arbitrary real values and produces another N-dimensional vector
+with real values in the range (0, 1) that add up to 1.0. It maps :math:`\mathbb{R}^N \rightarrow \mathbb{R}^N`
+
+.. math::
+
+     \sigma(\boldsymbol{o}): \begin{pmatrix}o_1\\o_2\\\dots\\o_n\end{pmatrix} \rightarrow \begin{pmatrix}\sigma_1\\\sigma_2\\\dots\\\sigma_n\end{pmatrix}
+
+This property of softmax function that it outputs a probability distribution makes it suitable for probabilistic
+interpretation in classification tasks. Neural networks, however, are commonly trained under a log loss (or
+cross-entropy) regime
+
+We are going to compute the derivative of the softmax function because we will be using it for training our RNN model
+shortly. But before diving in, it is important to keep in mind that Softmax is fundamentally a vector function. It takes
+a vector as input and produces a vector as output; in other words, it has multiple inputs and multiple outputs.
+Therefore, we cannot just ask for "the derivative of softmax"; We should instead specify:
+
+1. Which component (output element) of softmax we're seeking to find the derivative of.
+2. Since softmax has multiple inputs, with respect to which input element the partial derivative is computed.
+
+What we're looking for is the partial derivatives of
+
+.. math::
+
+    \frac{\partial \sigma_i}{\partial o_k} = \frac{\partial }{\partial o_k} \frac{e^{o_i}}{\sum_{j = 1}^ne^{o_j}}
+
+
+:math:`\frac{\partial \sigma_i}{\partial o_k}` **is the partial derivative of the i-th output with respect with the
+k-th input**.
+
+We'll be using the quotient rule of derivatives. For :math:`h(x) = \frac{f(x)}{g(x)}` where both :math:`f` and :math:`g`
+are differentiable and :math:`g(x) \ne 0`, The `quotient rule <https://en.wikipedia.org/wiki/Quotient_rule>`_ states
+that the derivative of :math:`h(x)` is
+
+.. math::
+
+    h'(x) = \frac{f'(x)g(x) - f(x)g'(x)}{g^2(x)}
+
+In our case, we have
+
+.. math::
+
+    f'(o_k) = \frac{\partial}{\partial o_k} e^{o_i} = \begin{cases}
+                                                          e^{o_k}, & \text{if}\ i = k \\
+                                                          0,       & \text{otherwise}
+                                                      \end{cases}
+
+.. math::
+
+    g'(o_k) = \frac{\partial}{\partial o_k} \sum_{j = 1}^ne^{o_j} = \left( \frac{\partial e^{o_1}}{\partial o_k} + \frac{\partial e^{o_2}}{\partial o_k} + \dots + \frac{\partial e^{o_k}}{\partial o_k} + \dots + \frac{\partial e^{o_n}}{\partial o_k} \right) = \frac{\partial e^{o_k}}{\partial o_k} = e^{o_k}
+
+The rest of it becomes trivial then. When :math:`i = k`,
+
+.. math::
+
+    \frac{\partial \sigma_i}{\partial o_k} = \frac{e^{o_k} \sum_{j = 1}^ne^{o_j} - e^{o_k} e^{o_i}}{\left( \sum_{j = 1}^ne^{o_j} \right)^2}
+                                           = \frac{e^{o_i} \sum_{j = 1}^ne^{o_j} - e^{o_i} e^{o_i}}{\left( \sum_{j = 1}^ne^{o_j} \right)^2}
+                                           = \frac{e^{o_i}}{\sum_{j = 1}^ne^{o_j}} \frac{\sum_{j = 1}^ne^{o_j} - e^{o_i}}{\sum_{j = 1}^ne^{o_j}} \\
+
+                                           = \sigma_i\left( \frac{\sum_{j = 1}^ne^{o_j}}{\sum_{j = 1}^ne^{o_j}} - \frac{e^{o_i}}{\sum_{j = 1}^ne^{o_j}} \right)
+                                           = \sigma_i \left( 1 - \sigma_i \right)
+
+When :math:`i \ne k`
+
+.. math::
+
+    \frac{\partial \sigma_i}{\partial o_k} = \frac{-e^{o_k} e^{o_i}}{\left( \sum_{j = 1}^ne^{o_j} \right)^2} = -\sigma_i\sigma_k
+
+This concludes the derivative of the softmax function:
+
+.. math::
+
+    \frac{\partial \sigma_i}{\partial o_k} = \begin{cases}
+                                                 \sigma_i \left( 1 - \sigma_i \right), & \text{if}\ i = k \\
+                                                 -\sigma_i\sigma_k,                    & \text{otherwise}
+                                             \end{cases}
+
+Cross-Entropy
+"""""""""""""
+
+.. admonition:: Cross-Entropy `Wikipedia <https://en.wikipedia.org/wiki/Cross-entropy>`_
+
+    In information theory, the cross-entropy between two probability distributions :math:`p` and :math:`q` over the same
+    underlying set of events measures the average number of bits needed to identify an event drawn from the set if a
+    coding scheme used for the set is optimized for an estimated probability distribution :math:`q`, rather than the
+    true distribution :math:`p`
+
+Confused? Let's put it in the context of Machine Learning.
+
+Machine Learning sees the world based on probability. The "probability distribution" identifies the various tasks to
+learn. For example, a daily language such as English or Chinese, can be seen as a probability distribution. The
+probability of "name" followed by "is" is far greater than "are" as in "My name is Jack". We call such language
+distribution :math:`p`. The task of RNN (or Machine Learning in general) is to learn an approximated distribution of
+:math:`p`; we call this approximation :math:`q`
+
+"The average number of bits needed" is can be seen as the distance between :math:`p` and :math:`q` given an event. In
+analogy of language, this can be the *quantitative* measure of the deviation between a real language phrase
+"My name is Jack" and "My name are Jack".
+
+At this point, it is easy to image that, in the Machine Learning world, the cross entropy indicates the distance between
+what the model believes the output distribution should be and what the original distribution really is.
+
+Now we have an intuitive understanding of cross entropy, let's formally define it.
+
+The cross-entropy of the discrete probability distribution :math:`q` relative to a distribution :math:`p` over a given
+set is defined as
+
+.. math::
+
+    H(p, q) = -\sum_x p(x)\log q(x)
+
+In RNN, the probability distribution of :math:`q(x)` is exactly the softmax function we defined earlier:
+
+.. math::
+
+    \mathcal{L} = -\sum_i p(i)\log\sigma(\boldsymbol{o})_i = -\sum_i \log\sigma(\boldsymbol{o})_i = -\log\boldsymbol{\hat{y}}^{(t)}
+
+where
+
+- :math:`\boldsymbol{o}` is the predicted sequence by RNN and :math:`o_i` is the i-th element of the predicted sequence
+
+.. admonition:: What is the Mathematical form of :math:`p(i)` in RNN? Why would it become 1?
+
+    By definition, :math:`p(i)` is the *true* distribution whose exact functional form is unknown. In the language of
+    Approximation Theory, :math:`p(i)` is the function that RNN is trying to learn or approximate mathematically.
+
+    Although the :math:`p(i)` makes the exact form of :math:`\mathcal{L}` unknown, computationally :math:`p(i)` is
+    perfectly defined in each training example. Taking our "hello" example:
+
+    .. figure:: ../img/char-level-language-model.png
+        :align: center
+        :width: 60%
+
+    The 4 probability distributions of :math:`q(x)` is "reflected" in the **output layer** of this example. They are
+    "reflecting" the probability distribution of :math:`q(x)` because they are only :math:`o` values and have not been
+    transformed to the :math:`\sigma` distribution yet. But in this case, we are 100% sure that the true probability
+    distribution :math:`p(i)` for the 4 outputs are
+
+    .. math::
+
+        \begin{pmatrix}0\\1\\0\\0\end{pmatrix}, \begin{pmatrix}0\\0\\1\\0\end{pmatrix}, \begin{pmatrix}0\\0\\1\\0\end{pmatrix}, \begin{pmatrix}0\\0\\0\\1\end{pmatrix}
+
+    respectively. *That is all we need for calculating the* :math:`\mathcal{L}`
+
+Deriving Gradient Descent Weight Update Rule
+--------------------------------------------
+
+*Training a RNN model of is the same thing as searching for the optimal values for the following parameters of these two
+perceptrons*:
+
+1. :math:`W_{xh}`
+2. :math:`W_{hh}`
+3. :math:`W_{yh}`
+4. :math:`b_h`
+5. :math:`b_o`
+
+By the Gradient Descent discussed in `MACHINE LEARNING by Mitchell, Thom M. (1997)`_ tells us we should derive the
+weight updat rule by *taking partial derivatives with respect to all of the variables above*. Let's start with
+:math:`W_{yh}`
+
+`MACHINE LEARNING by Mitchell, Thom M. (1997)`_ has mentioned gradients and partial derivatives as being important for
+an optimization algorithm to update, say, the model weights of a neural network to reach an optimal set of weights. The
+use of partial derivatives permits each weight to be updated independently of the others, by calculating the gradient of
+the error curve with respect to each weight in turn.
+
+Many of the functions that we usually work with in machine learning are *multivariate*, *vector-valued* functions, which
+means that they map multiple real inputs :math:`n` to multiple real outputs :math:`m`:
+
+.. math::
+
+    f: \mathbb{R}^n \rightarrow \mathbb{R}^m
+
+In training a neural network, the backpropagation algorithm is responsible for sharing back the error calculated at the
+output layer among the neurons comprising the different hidden layers of the neural network, until it reaches the input.
+
+If our RNN contains only 1 perceptron unit, the error is propagated back by, using the
+`Chain Rule <https://en.wikipedia.org/wiki/Chain_rule>`_ of :math:`\frac{dz}{dx} = \frac{dz}{dy}\frac{dy}{dx}`:
+
+.. math::
+
+    \frac{\partial \mathcal{L}}{\partial W} = \frac{\partial \mathcal{L}}{\partial o}\frac{\partial o}{\partial W}
+
+Note that in the RNN mode, :math:`\mathcal{L}` is not a direct function of :math:`W`. Thus its first order derivative
+cannot be computed unless we connect the :math:`\mathcal{L}` to :math:`o` first and then to :math:`W`, because both the
+first order derivatives of :math:`\frac{\partial \mathcal{L}}{\partial o}` and :math:`\frac{\partial o}{\partial W}` are
+defined by the model
+
+It is more often the case that we'd have many connected perceptrons populating the network, each attributed a different
+weight. Since this is the case for RNN, we can generalise multiple inputs and multiple outputs using the **Generalized
+Chain Rule**:
+
+Consider the case where :math:`x \in \mathbb{R}^m` and :math:`u \in \mathbb{R}^n`; an inner function, :math:`f`, maps
+:math:`m` inputs to :math:`n` outputs, while an outer function, :math:`g`, receives :math:`n` inputs to produce an
+output, :math:`h \in \mathbb{R}^k`. For :math:`i = 1, \dots, m`  the generalized chain rule states:
+
+.. math::
+
+    \frac{\partial h}{\partial x_i} = \frac{\partial h}{\partial u_1} \frac{\partial u_1}{\partial x_i} + \frac{\partial h}{\partial u_2} \frac{\partial u_2}{\partial x_i} + \dots + \frac{\partial h}{\partial u_n} \frac{\partial u_n}{\partial x_i} = \sum_{j = 1}^n \frac{\partial h}{\partial u_j} \frac{\partial u_j}{\partial x_i}
+
+Therefore, the error propagation of Gradient Descent in RNN is
+
+.. math::
+
+    \color{green} \boxed{
+        \begin{align}
+            \frac{\partial \mathcal{L}}{\partial W_{yh}} = \sum_{t = 1}^\tau \sum_{i = 1}^n \frac{\partial \mathcal{L}}{\partial o_i^{(t)}} \frac{\partial o_i^{(t)}}{\partial W_{yh}} \\ \\
+            \frac{\partial \mathcal{L}}{\partial W_{hh}} = \sum_{t = 1}^\tau \sum_{i = 1}^n \frac{\partial \mathcal{L}}{\partial h_i^{(t)}} \frac{\partial h_i^{(t)}}{\partial W_{hh}} \\ \\
+            \frac{\partial \mathcal{L}}{\partial W_{xh}} = \sum_{t = 1}^\tau \sum_{i = 1}^n \frac{\partial \mathcal{L}}{\partial h_i^{(t)}} \frac{\partial h_i^{(t)}}{\partial W_{xh}}
+        \end{align}
+    }
+
+where :math:`n` is the length of a RNN sequence and :math:`t` is the index of timestep
+
+.. admonition:: :math:`\sum_{t = 1}^\tau`
+
+    We assume the error is the sum of all errors of each timestep, which is why we include the :math:`\sum_{t = 1}^\tau`
+    term
+
+Let's look at :math:`\frac{\partial \mathcal{L}}{W_{yh}}` first
+
+.. math::
+
+    \frac{\partial \mathcal{L}}{W_{yh}} = \sum_{t = 1}^\tau \sum_{i = 1}^n \frac{\partial \mathcal{L}}{\partial o_i^{(t)}} \frac{\partial o_i^{(t)}}{\partial W_{yh}}
+
+Since :math:`o_i = \left( W_{yh}h_i + b_o \right)`,
+
+.. math::
+
+    \frac{\partial o_i}{W_{yh}} = \frac{\partial }{W_{yh}}\left( W_{yh}h_i + b_o \right) = h_i
+
+For the :math:`\frac{\partial \mathcal{L}}{\partial o_i}` we shall recall from the earlier discussion on softmax
+derivative that we cannot simply have
+
+.. math::
+
+    \frac{\partial \mathcal{L}}{\partial o_i} = -\frac{\partial}{\partial o_i}\sum_i^np(i)\log\sigma_i
+
+because we need to
+
+1. specify which component (output element) we're seeking to find the derivative of
+2. with respect to which input element the partial derivative is computed
+
+Therefore:
+
+.. math::
+
+    \frac{\partial \mathcal{L}}{\partial o_i} = -\frac{\partial}{\partial o_i}\sum_j^np(j)\log\sigma_j = -\sum_j^n\frac{\partial}{\partial o_i}p(j)\log\sigma_j = -\sum_j^np(j)\frac{\partial \log\sigma_j}{\partial o_i}
+
+where :math:`n` is the number of timesteps (or the length of a sequence such as "hell")
+
+Applying the chain rule again:
+
+.. math::
+
+    -\sum_j^np(j)\frac{\partial \log\sigma_j}{\partial o_i} = -\sum_j^np(j)\frac{1}{\sigma_j}\frac{\partial\sigma_j}{\partial o_i}
+
+Recall we have already derived that
+
+.. math::
+
+    \frac{\partial \sigma_i}{\partial o_j} = \begin{cases}
+                                                 \sigma_i \left( 1 - \sigma_i \right), & \text{if}\ i = j \\
+                                                 -\sigma_i\sigma_j,                    & \text{otherwise}
+                                             \end{cases}
+
+.. math::
+
+    -\sum_j^np(j)\frac{1}{\sigma_j}\frac{\partial\sigma_j}{\partial o_i} = -\sum_{i = j}^np(j)\frac{1}{\sigma_j}\frac{\partial\sigma_j}{\partial o_i} -\sum_{i \ne j}^np(j)\frac{1}{\sigma_j}\frac{\partial\sigma_j}{\partial o_i} = -p(i)(1 - \sigma_i) + \sum_{i \ne j}^np(j)\sigma_i
+
+Observing that
+
+.. math::
+
+    \sum_{j}^np(j) = 1
+
+.. math::
+
+    -p(i)(1 - \sigma_i) + \sum_{i \ne j}^np(j)\sigma_i = -p(i) + p(i)\sigma_i + \sum_{i \ne j}^np(j)\sigma_i = \sigma_i - p(i)
+
+.. math::
+
+    \color{green} \boxed{\frac{\partial \mathcal{L}}{\partial o_i} = \sigma_i - p(i)}
+
+.. math::
+
+    \color{green} \boxed{ \frac{\partial \mathcal{L}}{\partial W_{yh}} = \sum_{t = 1}^\tau \sum_i^n\left[ \sigma_i - p(i) \right] h_i = \sum_{t = 1}^\tau \left( \boldsymbol{\sigma} - \boldsymbol{p} \right) \boldsymbol{h}^{(t)} }
+
+.. math::
+
+    \frac{\partial \mathcal{L}}{b_o} = \sum_{t = 1}^\tau \sum_i^n\frac{\partial \mathcal{L}}{\partial o_i^{(t)}}\frac{\partial o_i^{(t)}}{\partial b_o^{(t)}} = \sum_{t = 1}^\tau \sum_i^n\left[ \sigma_i - p(i) \right] \times 1
+
+.. math::
+
+    \color{green} \boxed{ \frac{\partial \mathcal{L}}{\partial b_o} = \sum_{t = 1}^\tau \sum_i^n\left[ \sigma_i - p(i) \right] = \sum_{t = 1}^\tau \boldsymbol{\sigma} - \boldsymbol{p} }
+
+We have at this point derived backpropagating rule for :math:`W_{yh}` and :math:`b_o`:
+
+1. :math:`W_{xh}`
+2. :math:`W_{hh}`
+3. ✅ :math:`W_{yh}`
+4. :math:`b_h`
+5. ✅ :math:`b_o`
+
+Now let's look at :math:`\frac{\partial \mathcal{L}}{\partial W_{hh}}`:
+
+Recall from *Deep Learning*, section 6.5.2, p. 207 that the vector notation of
+:math:`\frac{\partial z}{\partial x_i} = \sum_j \frac{\partial z}{\partial y_j}\frac{\partial y_j}{\partial x_i}` is
+
+.. math::
+
+    \nabla_{\boldsymbol{x}}z = \left( \frac{\partial \boldsymbol{y}}{\partial \boldsymbol{x}} \right)^\intercal \nabla_{\boldsymbol{y}}z
+
+This gives us a start with:
+
+.. math::
+
+    \begin{align}
+        \frac{\partial \mathcal{L}}{\partial W_{hh}} &= \sum_{t = 1}^\tau \sum_{i = 1}^n \frac{\partial \mathcal{L}}{\partial h_i^{(t)}} \frac{\partial h_i^{(t)}}{\partial W_{hh}} \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal \nabla_{\boldsymbol{W_{hh}}}\boldsymbol{h}^{(t)} \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{W_{hh}}} \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\boldsymbol{h}^{(t)} \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{W_{hh}}} \right)^\intercal \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{W_{hh}}} \right)^\intercal \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{h}^{(t - 1)}} \right)^\intercal \left( \frac{\partial \boldsymbol{h}^{(t - 1)}}{\partial \boldsymbol{W_{hh}}} \right)^\intercal \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{h}^{(t - 1)}} \right)^\intercal \left( \frac{\partial \boldsymbol{h}^{(t - 1)}}{\partial \boldsymbol{h}^{(t)}}\frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{h}^{(t)}}\frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{W_{hh}}} \right)^\intercal \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{h}^{(t - 1)}} \right)^\intercal \left( \frac{\partial \boldsymbol{h}^{(t - 1)}}{\partial \boldsymbol{h}^{(t)}}\frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{W_{hh}}}\frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{h}^{(t - 1)}} \right)^\intercal \left( \frac{\partial \boldsymbol{h}^{(t - 1)}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal  \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{W_{hh}}} \right)^\intercal \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \\
+        & = \sum_{t = 1}^\tau \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{W_{hh}}} \right)^\intercal \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal \frac{\partial \mathcal{L}}{\partial \boldsymbol{h}^{(t)}} \\
+        & = \sum_{t = 1}^\tau diag\left[ 1 - \left(\boldsymbol{h}^{(t)}\right)^2 \right] \boldsymbol{h}^{(t - 1)} \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \\
+        & = \sum_{t = 1}^\tau diag\left[ 1 - \left(\boldsymbol{h}^{(t)}\right)^2 \right] \left( \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \right) {\boldsymbol{h}^{(t - 1)}}^\intercal
+    \end{align}
+
+.. math::
+
+    \color{green} \boxed{ \frac{\partial \mathcal{L}}{\partial W_{hh}} = \sum_{t = 1}^\tau diag\left[ 1 - \left(\boldsymbol{h}^{(t)}\right)^2 \right] \left( \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \right) {\boldsymbol{h}^{(t - 1)}}^\intercal }
+
+The equation above leaves us with a term :math:`\nabla_{\boldsymbol{h}^{(t)}}\mathcal{L}`, which we calculate next. Note
+that the back propagation on :math:`\boldsymbol{h}^{(t)}` has source from both :math:`\boldsymbol{o}^{(t)}` and
+:math:`\boldsymbol{h}^{(t + 1)}`. It's gradient, therefore, is given by
+
+.. math::
+
+    \begin{align}
+        \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} &= \left( \frac{\partial \boldsymbol{o}^{(t)}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal \nabla_{\boldsymbol{o}^{(t)}}\mathcal{L} + \left( \frac{\partial \boldsymbol{h}^{(t + 1)}}{\partial \boldsymbol{h}^{(t)}} \right)^\intercal \nabla_{\boldsymbol{h}^{(t + 1)}}\mathcal{L} \\
+        &= \left( \boldsymbol{W_{yh}} \right)^\intercal \nabla_{\boldsymbol{o}^{(t)}}\mathcal{L} + \left( diag\left[ 1 - (\boldsymbol{h}^{(t + 1)})^2 \right] \boldsymbol{W_{hh}} \right)^\intercal \nabla_{\boldsymbol{h}^{(t + 1)}}\mathcal{L} \\
+        &= \left( \boldsymbol{W_{yh}} \right)^\intercal \nabla_{\boldsymbol{o}^{(t)}}\mathcal{L}+ \boldsymbol{W_{hh}}^\intercal \nabla_{\boldsymbol{h}^{(t + 1)}}\mathcal{L} \left( diag\left[ 1 - (\boldsymbol{h}^{(t + 1)})^2 \right] \right)
+    \end{align}
+
+.. math::
+
+    \color{green} \boxed{ \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} = \left( \boldsymbol{W_{yh}} \right)^\intercal \nabla_{\boldsymbol{o}^{(t)}}\mathcal{L} + \boldsymbol{W_{hh}}^\intercal \nabla_{\boldsymbol{h}^{(t + 1)}}\mathcal{L} \left( diag\left[ 1 - (\boldsymbol{h}^{(t + 1)})^2 \right] \right) }
+
+Note that the 2nd term
+:math:`\boldsymbol{W_{xh}}^\intercal \nabla_{\boldsymbol{h}^{(t + 1)}}\mathcal{L} \left( diag\left[ 1 - (\boldsymbol{h}^{(t + 1)})^2 \right] \right)`
+is zero at first iteration propagating back because for the last-layer (unrolled) of RNN , there's no gradient update
+flow from the next hidden state.
+
+So far we have derived backpropagating rule for :math:`W_{hh}`
+
+1. :math:`W_{xh}`
+2. ✅ :math:`W_{hh}`
+3. ✅ :math:`W_{yh}`
+4. :math:`b_h`
+5. ✅ :math:`b_o`
+
+Let's tackle the remaining  :math:`\frac{\partial \mathcal{L}}{\partial W_{xh}}` and :math:`b_h`:
+
+.. math::
+
+    \begin{align}
+        \frac{\partial \mathcal{L}}{\partial W_{xh}} &= \sum_{t = 1}^\tau \sum_{i = 1}^n \frac{\partial \mathcal{L}}{\partial h_i^{(t)}} \frac{\partial h_i^{(t)}}{\partial W_{xh}} \\
+        &= \sum_{t = 1}^\tau \left( \frac{\partial \boldsymbol{h}^{(t)}}{\partial \boldsymbol{W_{xh}}} \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \\
+        &= \sum_{t = 1}^\tau \left( diag\left[ 1 - (\boldsymbol{h}^{(t)})^2 \right] \boldsymbol{x}^{(t)} \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \\
+        &= \sum_{t = 1}^\tau \left( diag\left[ 1 - (\boldsymbol{h}^{(t)})^2 \right] \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \left( \boldsymbol{x}^{(t)} \right)
+    \end{align}
+
+.. math::
+
+    \color{green} \boxed{ \frac{\partial \mathcal{L}}{\partial W_{xh}} = \sum_{t = 1}^\tau \left( diag\left[ 1 - (\boldsymbol{h}^{(t)})^2 \right] \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \left( \boldsymbol{x}^{(t)} \right) }
+
+.. math::
+
+    \begin{align}
+        \frac{\partial \mathcal{L}}{\partial b_h} &= \sum_{t = 1}^\tau \sum_{i = 1}^n \frac{\partial \mathcal{L}}{\partial h_i^{(t)}} \frac{\partial h_i^{(t)}}{\partial b_h^{(t)}} \\
+        &= \sum_{t = 1}^\tau \left( \frac{\partial h_i^{(t)}}{\partial b_h^{(t)}} \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \\
+        &= \sum_{t = 1}^\tau \left( diag\left[ 1 - (\boldsymbol{h}^{(t)})^2 \right] \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L}
+    \end{align}
+
+.. math::
+
+    \color{green} \boxed{ \frac{\partial \mathcal{L}}{\partial b_h} = \sum_{t = 1}^\tau \left( diag\left[ 1 - (\boldsymbol{h}^{(t)})^2 \right] \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} }
+
+This concludes our propagation rules for training RNN:
+
+.. math::
+
+    \color{green} \boxed{
+        \begin{gather*}
+            \frac{\partial \mathcal{L}}{\partial W_{xh}} = \sum_{t = 1}^\tau \left( diag\left[ 1 - (\boldsymbol{h}^{(t)})^2 \right] \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \left( \boldsymbol{x}^{(t)} \right) \\ \\
+            \frac{\partial \mathcal{L}}{\partial W_{hh}} = \sum_{t = 1}^\tau diag\left[ 1 - \left(\boldsymbol{h}^{(t)}\right)^2 \right] \left( \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \right) {\boldsymbol{h}^{(t - 1)}}^\intercal \\ \\
+            \frac{\partial \mathcal{L}}{\partial W_{yh}} = \sum_{t = 1}^\tau \left( \boldsymbol{\sigma} - \boldsymbol{p} \right) \boldsymbol{h}^{(t)} \\ \\
+            \frac{\partial \mathcal{L}}{\partial b_h} = \sum_{t = 1}^\tau \left( diag\left[ 1 - (\boldsymbol{h}^{(t)})^2 \right] \right)^\intercal \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} \\ \\
+            \frac{\partial \mathcal{L}}{\partial b_o} =\sum_{t = 1}^\tau \boldsymbol{\sigma} - \boldsymbol{p}
+        \end{gather*}
+    }
+
+where
+
+.. math::
+
+    \color{green} \boxed{ \nabla_{\boldsymbol{h}^{(t)}}\mathcal{L} = \left( \boldsymbol{W_{yh}} \right)^\intercal \nabla_{\boldsymbol{o}^{(t)}}\mathcal{L}+ \boldsymbol{W_{hh}}^\intercal \nabla_{\boldsymbol{h}^{(t + 1)}}\mathcal{L} \left( diag\left[ 1 - (\boldsymbol{h}^{(t + 1)})^2 \right] \right) }
+
+Computational Gradient Descent Weight Update Rule
+-------------------------------------------------
+
+What does the propagation rules above look like in Python?
+
+Example
+-------
+
+`Pride and Prejudice by Jane Austen <https://www.gutenberg.org/ebooks/1342>`_
+
+
+.. code-block:: python
+
+
+
+
+
+
+
+
+
+.. _`exploding gradient`: https://qubitpi.github.io/stanford-cs231n.github.io/rnn/#vanilla-rnn-gradient-flow--vanishing-gradient-problem
+
+.. _`MACHINE LEARNING by Mitchell, Thom M. (1997)`: https://a.co/d/bjmsEOg
+
+.. _`loss function`: https://qubitpi.github.io/stanford-cs231n.github.io/neural-networks-2/#losses
+.. _`LSTM Formulation`: https://qubitpi.github.io/stanford-cs231n.github.io/rnn/#lstm-formulation
+
+.. _`Vanilla RNN Gradient Flow & Vanishing Gradient Problem`: https://qubitpi.github.io/stanford-cs231n.github.io/rnn/#vanilla-rnn-gradient-flow--vanishing-gradient-problem

docs/source/rnn/vanilla.rst

@@ -1,176 +0,0 @@
-================================================
-Introduction to Recurrent Neural Networks (RNNs)
-================================================
-
-.. contents:: Table of Contents
-    :depth: 2
-
-
-Mathematical Formulation
-------------------------
-
-Recurrent neural networks, also known as RNNs, are a class of neural networks that allow previous outputs to be used as
-inputs while having hidden states. They are typically as follows:
-
-.. figure:: ../img/architecture-rnn-ltr.png
-    :align: center
-
-For each timestep :math:`t` the activation :math:`a^{\langle t \rangle}` and the output :math:`y^{\langle t \rangle}` are expressed as follows:
-
-.. math::
-
-    h^{\langle t \rangle} = g_1\left( W_{hh}h^{\langle t - 1 \rangle} + W_{hx}x^{\langle t \rangle} + b_h \right)
-
-    y^{\langle t \rangle} = g_2\left( W_{yh}h^{\langle t \rangle} + b_y \right)
-
-where :math:`W_{hx}`, :math:`W_{hh}`, :math:`W_{yh}`, :math:`b_h`, :math:`b_y` are coefficients that are shared temporally and :math:`g_1`, :math:`g_2` are activation functions.
-
-.. figure:: ../img/description-block-rnn-ltr.png
-    :align: center
-
-A Python implementation of network above, as an example, could be as follows:
-
-.. code-block:: python
-
-    import numpy as np
-    from math import exp
-
-    np.random.seed(0)
-    class VanillaRecurrentNetwork(object):
-
-        def __init__(self):
-            self.hidden_state = np.zeros((3, 3))
-            self.W_hh = np.random.randn(3, 3)
-            self.W_xh = np.random.randn(3, 3)
-            self.W_hy = np.random.randn(3, 3)
-            self.Bh = np.random.randn(3,)
-            self.By = np.random.rand(3,)
-
-            self.hidden_state_activation_function = lambda x : np.tanh(x)
-            self.y_activation_function = lambda x : x
-
-        def forward_prop(self, x):
-            self.hidden_state = self.hidden_state_activation_function(
-                np.dot(self.hidden_state, self.W_hh) + np.dot(x, self.W_xh) + self.Bh
-            )
-
-            return self.y_activation_function(self.W_hy.dot(self.hidden_state) + self.By)
-
-Notice the weight matrix above are randomly initialized. This makes it a "silly" network that doesn't help us anything
-good:
-
-.. code-block:: python
-
-    input_vector = np.ones((3, 3))
-    silly_network = RecurrentNetwork()
-
-    # Notice that same input, but leads to different ouptut at every single time step.
-    print silly_network.forward_prop(input_vector)
-    print silly_network.forward_prop(input_vector)
-    print silly_network.forward_prop(input_vector)
-
-    # this gives us
-    [[-1.73665315 -2.40366542 -2.72344361]
-     [ 1.61591482  1.45557046  1.13262256]
-     [ 1.68977504  1.54059305  1.21757531]]
-    [[-2.15023381 -2.41205828 -2.71701457]
-     [ 1.71962883  1.45767515  1.13101034]
-     [ 1.80488553  1.542929    1.21578594]]
-    [[-2.15024751 -2.41207375 -2.720968  ]
-     [ 1.71963227  1.45767903  1.13200175]
-     [ 1.80488935  1.54293331  1.21688628]]
-
-This is because we haven't train our RNN network yet, which we discuss next
-
-Training
---------
-
-.. admonition:: Prerequisite
-
-    We would assume some basic Artificial Neural Network concepts, which are drawn from *Chapter 4 - Artificial Neural
-    Networks* (p. 81) of `MACHINE LEARNING by Mitchell, Thom M. (1997)`_ Paperback. Please, if possible, read the
-    chapter beforehand and refer to it if something looks confusing in the discussion of this section
-
-In the case of a recurrent neural network, we are essentially backpropagation through time, which means that we are
-forwarding through entire sequence to compute losses, then backwarding through entire sequence to compute gradients.
-Formally, the `loss function`_ :math:`\mathcal{L}` of all time steps is defined as the sum of
-the loss at every time step:
-
-.. math::
-
-    \mathcal{L}\left( \hat{y}, y \right) = \sum_{t = 1}^{T_y}\mathcal{L}\left( \hat{y}^{<t>}, y^{<t>} \right)
-
-However, this becomes problematic when we want to train a sequence that is very long. For example, if we were to train a
-a paragraph of words, we have to iterate through many layers before we can compute one simple gradient step. In
-practice, for the back propagation, we examine how the output at the very *last* timestep affects the weights at the
-very first time step. Then we can compute the gradient of loss function, the details of which can be found in the
-`Vanilla RNN Gradient Flow & Vanishing Gradient Problem`_
-
-.. admonition:: Gradient Clipping
-
-    Gradient clipping is a technique used to cope with the `exploding gradient`_ problem sometimes encountered when
-    performing backpropagation. By capping the maximum value for the gradient, this phenomenon is controlled in
-    practice.
-
-    .. figure:: ../img/gradient-clipping.png
-        :align: center
-
-    In order to remedy the vanishing gradient problem, specific gates are used in some types of RNNs and usually have a
-    well-defined purpose. They are usually noted :math:`\Gamma` and are defined as
-
-    .. math::
-
-        \Gamma = \sigma(Wx^{<t>} + Ua^{<t - 1>} + b)
-
-    where :math:`W`, :math:`U`, and :math:`b` are coefficients specific to the gate and :math:`\sigma` is the sigmoid
-    function
-
-LSTM Formulation
-^^^^^^^^^^^^^^^^
-
-Now we know that Vanilla RNN has Vanishing/exploding gradient problem, `LSTM Formulation`_ discusses the theory of LSTM
-which is used to remedy this problem.
-
-Applications of RNNs
---------------------
-
-RNN models are mostly used in the fields of natural language processing and speech recognition. The different
-applications are summed up in the table below:
-
-.. list-table:: Applications of RNNs
-   :widths: 20 60 20
-   :align: center
-   :header-rows: 1
-
-   * - Type of RNN
-     - Illustration
-     - Example
-   * - | One-to-one
-       | :math:`T_x = T_y = 1`
-     - .. figure:: ../img/rnn-one-to-one-ltr.png
-     - Traditional neural network
-   * - | One-to-many
-       | :math:`T_x = 1`, :math:`T_y > 1`
-     - .. figure:: ../img/rnn-one-to-many-ltr.png
-     - Music generation
-   * - | Many-to-one
-       | :math:`T_x > 1`, :math:`T_y = 1`
-     - .. figure:: ../img/rnn-many-to-one-ltr.png
-     - Sentiment classification
-   * - | Many-to-many
-       | :math:`T_x = T_y`
-     - .. figure:: ../img/rnn-many-to-many-same-ltr.png
-     - Named entity recognition
-   * - | Many-to-many
-       | :math:`T_x \ne T_y`
-     - .. figure:: ../img/rnn-many-to-many-different-ltr.png
-     - Machine translation
-
-.. _`exploding gradient`: https://qubitpi.github.io/stanford-cs231n.github.io/rnn/#vanilla-rnn-gradient-flow--vanishing-gradient-problem
-
-.. _`MACHINE LEARNING by Mitchell, Thom M. (1997)`: https://a.co/d/bjmsEOg
-
-.. _`loss function`: https://qubitpi.github.io/stanford-cs231n.github.io/neural-networks-2/#losses
-.. _`LSTM Formulation`: https://qubitpi.github.io/stanford-cs231n.github.io/rnn/#lstm-formulation
-
-.. _`Vanilla RNN Gradient Flow & Vanishing Gradient Problem`: https://qubitpi.github.io/stanford-cs231n.github.io/rnn/#vanilla-rnn-gradient-flow--vanishing-gradient-problem

lamassu/rnn/example.py

@@ -0,0 +1,50 @@
+import numpy as np
+
+from lamassu.rnn.rnn import Config
+from lamassu.rnn.rnn import RecurrentNeuralNetwork
+
+if __name__ == "__main__":
+    num_hidden_perceptrons= 100
+    seq_length = 25
+    learning_rate = 1e-1
+
+
+    data = open('pride-and-prejudice.txt', 'r').read()
+    char_set = list(set(data))
+    num_chars, num_unique_chars = len(data), len(char_set)
+    char_to_idx = { ch:i for i,ch in enumerate(char_set) }
+    idx_to_char = { i:ch for i,ch in enumerate(char_set) }
+
+    rnn = RecurrentNeuralNetwork(
+        Config(
+            num_hidden_perceptrons=num_hidden_perceptrons,
+            input_size=num_unique_chars,
+            learning_rate=learning_rate
+        )
+    )
+
+    num_iter, pointer = 0, 0
+
+
+    while True:
+        if pointer + seq_length + 1 >= len(data) or num_iter == 0:
+            prev_history = np.zeros((num_hidden_perceptrons, 1))
+            pointer = 0
+        input = [char_to_idx[c] for c in data[pointer: pointer + seq_length]]
+        target = [char_to_idx[c] for c in data[pointer + 1: pointer + seq_length + 1]]
+
+        if num_iter % 100 == 0: # inference after every 100 trainings
+            inferenced_idxes = rnn.inference(prev_history, input[0])
+            inferenced = ''.join(idx_to_char[idx] for idx in inferenced_idxes)
+            print("============ inference ============")
+            print(inferenced)
+
+        history, q, x, loss = rnn.forward_pass(input, target, prev_history)
+
+        if num_iter % 100 == 0:
+            print("loss: {}".format(loss))
+
+        prev_history = rnn.back_propagation(input, target, history, q, x)
+
+        pointer += seq_length
+        num_iter += 1

lamassu/rnn/rnn.py

@@ -0,0 +1,114 @@
+import numpy as np
+from math import exp
+from dataclasses import dataclass
+
+
+np.random.seed(0)
+
+@dataclass
+class Config():
+    num_hidden_perceptrons: int
+    input_size: int
+    learning_rate: float
+
+
+class RecurrentNeuralNetwork(object):
+    """
+    Architecture is single-hidden-layer
+    """
+
+    def __init__(self, config: Config):
+        self.config = config
+
+        self.W_xh = np.random.randn(config.num_hidden_perceptrons, config.input_size)
+        self.W_hh = np.random.randn(config.num_hidden_perceptrons, config.num_hidden_perceptrons)
+        self.W_yh = np.random.randn(config.input_size, config.num_hidden_perceptrons)
+
+        self.b_h = np.zeros((config.num_hidden_perceptrons, 1))
+        self.b_o = np.zeros((config.input_size, 1))
+
+    def forward_pass(self, input, target, prev_history):
+        """
+
+        :param input:  The input vector; each element is an index
+        :return:
+        """
+
+        history, x, o, q, loss = {}, {}, {}, {}, 0
+        history[-1] = np.copy(prev_history)
+
+        for t in range(len(input)):
+            x[t] = np.zeros((self.config.input_size, 1))
+            x[t][input[t]] = 1
+
+            if t == 0:
+                np.dot(self.W_hh, history[t - 1])
+                np.dot(self.W_xh, x[t])
+
+            history[t] = np.tanh(
+                np.dot(self.W_hh, history[t - 1]) + np.dot(self.W_xh, x[t]) + self.b_h
+            )
+            o[t] = np.dot(self.W_yh, history[t]) + self.b_o
+            q[t] = np.exp(o[t]) / np.sum(np.exp(o[t]))
+            loss += -np.log(q[t][target, 0])
+
+        return history, q, x, loss
+
+    def back_propagation(self, input, target, history, q, x):
+        gradient_loss_over_W_xh = np.zeros_like(self.W_xh)
+        gradient_loss_over_W_hh = np.zeros_like(self.W_hh)
+        gradient_loss_over_W_yh = np.zeros_like(self.W_yh)
+
+        gradient_loss_over_b_h = np.zeros_like(self.b_h)
+        gradient_loss_over_b_y = np.zeros_like(self.b_o)
+
+        gradient_loss_over_next_h = np.zeros_like(history[0])
+
+        for t in reversed(range(len(input))):
+            gradient_loss_over_o = np.copy(q[t])
+            gradient_loss_over_o[target[t]] -= 1
+
+            gradient_loss_over_W_yh += np.dot(gradient_loss_over_o, history[t].T)
+            gradient_loss_over_b_y += gradient_loss_over_o #
+
+            gradient_loss_over_h = np.dot(self.W_yh.T, gradient_loss_over_o) + gradient_loss_over_next_h
+            diag_times_gradient_loss_over_h = (1 - history[t] * history[t]) * gradient_loss_over_h
+
+            gradient_loss_over_b_h += diag_times_gradient_loss_over_h #
+
+            gradient_loss_over_W_xh += np.dot(diag_times_gradient_loss_over_h, x[t].T) #
+            gradient_loss_over_W_hh += np.dot(diag_times_gradient_loss_over_h, history[t - 1].T) #
+
+            gradient_loss_over_next_h = np.dot(self.W_hh.T, diag_times_gradient_loss_over_h)
+
+        for gradient in [gradient_loss_over_W_xh, gradient_loss_over_W_hh, gradient_loss_over_W_yh, gradient_loss_over_b_h, gradient_loss_over_b_y]:
+            np.clip(gradient, -5, 5, out=gradient) # avoid exploding gradients
+
+        # update weights
+        for param, gradient in zip(
+                [self.W_xh, self.W_hh, self.W_yh, self.b_h, self.b_o],
+                [gradient_loss_over_W_xh, gradient_loss_over_W_hh, gradient_loss_over_W_yh, gradient_loss_over_b_h, gradient_loss_over_b_y]):
+            param += -self.config.learning_rate * gradient
+
+        return history[len(input) - 1]
+
+    def inference(self, history, seed_idx):
+        x = np.zeros((self.config.input_size, 1))
+        x[seed_idx] = 1
+        idxes = []
+
+        for timestep in range(200):
+            history = np.tanh(np.dot(self.W_xh, x) + np.dot(self.W_hh, history) + self.b_h)
+            o = np.dot(self.W_yh, history) + self.b_o
+            p = np.exp(o) / np.sum(np.exp(o))
+
+            next_idx = self._inference_single(p.ravel())
+
+            x[next_idx] = 1
+            idxes.append(next_idx)
+
+        return idxes
+
+
+    def _inference_single(self, probability_distribution):
+        return np.random.choice(range(self.config.input_size), p=probability_distribution)

lamassu/rnn/vanilla.py

@@ -1,25 +0,0 @@
-import numpy as np
-from math import exp
-
-np.random.seed(0)
-class VanillaRecurrentNetwork(object):
-
-    def __init__(self):
-        self.hidden_state = np.zeros((3, 3))
-        self.W_hh = np.random.randn(3, 3)
-        self.W_xh = np.random.randn(3, 3)
-        self.W_hy = np.random.randn(3, 3)
-        self.Bh = np.random.randn(3,)
-        self.By = np.random.rand(3,)
-
-        self.hidden_state_activation_function = lambda x : np.tanh(x)
-        self.y_activation_function = lambda x : x
-
-        self.loss_funciton = lambda y : exp(y) / np.sum(exp(y))
-
-    def forward_prop(self, x):
-        self.hidden_state = self.hidden_state_activation_function(
-            np.dot(self.hidden_state, self.W_hh) + np.dot(x, self.W_xh) + self.Bh
-        )
-
-        return self.y_activation_function(self.W_hy.dot(self.hidden_state) + self.By)

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="lamassu",
-    version="0.0.11",
+    version="0.0.12",
     description="Empowering individual to agnostically run machine learning algorithms to produce ad-hoc AI features",
     url="https://github.com/QubitPi/lamassu",
     author="Jiaqi liu",