Upload folder using huggingface_hub

.gitattributes

@@ -7,3 +7,12 @@ data/datasets/local/open-asssistant-conversations/text/near_dup/data-00000-of-00
 data/datasets/local/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/local/open-asssistant-conversations/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/local/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/local/imdb/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

data/datasets/local/imdb/config.yml

@@ -0,0 +1,10 @@
+name: imdb
+namespace: local
+settings:
+ preferred_embedding: gte-small
+ ui:
+ media_paths:
+ - text
+source:
+ source_name: huggingface
+ dataset_name: imdb

data/datasets/local/imdb/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cf3f121bae8b8d8c12af8bebe4cda35c2a84750470fff57ea37a4930c257d6f
+size 86160733

data/datasets/local/imdb/manifest.json

@@ -0,0 +1,21 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "data_schema": {
+ "fields": {
+ "text": {
+ "dtype": "string"
+ },
+ "label": {
+ "dtype": "string"
+ },
+ "__hfsplit__": {
+ "dtype": "string"
+ },
+ "__rowid__": {
+ "dtype": "string"
+ }
+ }
+ }
+}

data/datasets/local/imdb/settings.json

@@ -0,0 +1 @@
+{"ui": {"media_paths": [["text"]], "markdown_paths": []}, "preferred_embedding": "gte-small"}

data/datasets/local/imdb/text/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4659a623093a2ef1646885a6ecb6ef86c56c2dcd0b10900d7b46d193dfb69e7f
+size 691432464

data/datasets/local/imdb/text/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cbf4a5777b0cd1f8bb5061a6177b27cc0f5a8a6349c487c0c5c52fe60697d64
+size 10390846

data/datasets/local/imdb/text/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+ "files": [],
+ "parquet_id": "gte-small(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "gte-small": {
+ "repeated_field": {
+ "fields": {
+ "embedding": {
+ "dtype": "embedding"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "signal_name": "gte-small"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "gte-small"
+ },
+ "enriched_path": [
+ "text"
+ ],
+ "vector_store": "hnsw"
+}

data/datasets/local/imdb/text/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:526e8505beb2386e3ff30367968685fd2229f76af2c0c86d50afaa7da3018dbc
+size 7476546

data/datasets/local/imdb/text/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45651752b9f1178504ed253e070243e2782a79faf0e1272c3b9e3ba4ed8a717d
+size 3309640

data/datasets/local/imdb/text/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "lang_detection(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "lang_detection": {
+ "dtype": "string",
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/local/imdb/text/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1ccd12fc66d0c31a19554fcb5f442751807745e51c3a9336cec637525a422fc
+size 3916036

data/datasets/local/imdb/text/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "near_dup(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "near_dup": {
+ "fields": {
+ "cluster_id": {
+ "dtype": "uint32",
+ "categorical": true
+ }
+ },
+ "signal": {
+ "threshold": 0.75,
+ "signal_name": "near_dup"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "threshold": 0.75,
+ "signal_name": "near_dup"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/local/imdb/text/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f1f559281ca4e3efcafd4b10c51cbe2f5039d86ce95d3dc07156671fd8b824
+size 3313984

data/datasets/local/imdb/text/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "pii(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "pii": {
+ "fields": {
+ "emails": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "ip_addresses": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "secrets": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/local/imdb/text/spacy_ner/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19ce0e0966a4db29b7b862aa3fa87ef3b02997e57efcdd722023819caa1be7bb
+size 8483750

data/datasets/local/imdb/text/spacy_ner/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "spacy_ner(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "spacy_ner": {
+ "repeated_field": {
+ "fields": {
+ "label": {
+ "dtype": "string"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/local/imdb/text/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:672357a255fecf4e29604674ff3ceb11b6772d0388293f5267f608a6163faf49
+size 4404092

data/datasets/local/imdb/text/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "text_statistics(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "text_statistics": {
+ "fields": {
+ "num_characters": {
+ "dtype": "int32"
+ },
+ "readability": {
+ "dtype": "float32"
+ },
+ "log(type_token_ratio)": {
+ "dtype": "float32"
+ },
+ "frac_non_ascii": {
+ "dtype": "float32",
+ "bins": [
+ [
+ "Low",
+ null,
+ 0.15
+ ],
+ [
+ "Medium",
+ 0.15,
+ 0.3
+ ],
+ [
+ "High",
+ 0.3,
+ null
+ ]
+ ]
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}