Upload folder using huggingface_hub

.gitattributes

@@ -39,3 +39,12 @@ data/datasets/lilac/pile-of-law-r-legaladvice/text/near_dup/data-00000-of-00001.
 data/datasets/lilac/pile-of-law-r-legaladvice/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/pile-of-law-r-legaladvice/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/pile-of-law-r-legaladvice/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/open-asssistant-conversations/config.yml

@@ -0,0 +1,31 @@
+embeddings:
+- embedding: gte-small
+ path: text
+name: open-asssistant-conversations
+namespace: local
+settings:
+ preferred_embedding: gte-small
+ ui:
+ media_paths:
+ - text
+signals:
+- path: text
+ signal:
+ signal_name: text_statistics
+- path: text
+ signal:
+ signal_name: lang_detection
+- path: text
+ signal:
+ signal_name: near_dup
+- path: text
+ signal:
+ signal_name: spacy_ner
+- path: text
+ signal:
+ signal_name: pii
+source:
+ dataset_name: OpenAssistant/oasst1
+ source_name: huggingface
+tags:
+- machine-learning

data/datasets/lilac/open-asssistant-conversations/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d53dbedb539cf7fa3f89d739f698fd3ccf1fbbd86dac20bd0b74cf674cc508e8
+size 42071566

data/datasets/lilac/open-asssistant-conversations/manifest.json

@@ -0,0 +1,118 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "data_schema": {
+ "fields": {
+ "message_id": {
+ "dtype": "string"
+ },
+ "parent_id": {
+ "dtype": "string"
+ },
+ "user_id": {
+ "dtype": "string"
+ },
+ "created_date": {
+ "dtype": "string"
+ },
+ "text": {
+ "dtype": "string"
+ },
+ "role": {
+ "dtype": "string"
+ },
+ "lang": {
+ "dtype": "string"
+ },
+ "review_count": {
+ "dtype": "int32"
+ },
+ "review_result": {
+ "dtype": "boolean"
+ },
+ "deleted": {
+ "dtype": "boolean"
+ },
+ "rank": {
+ "dtype": "int32"
+ },
+ "synthetic": {
+ "dtype": "boolean"
+ },
+ "model_name": {
+ "dtype": "string"
+ },
+ "detoxify": {
+ "fields": {
+ "toxicity": {
+ "dtype": "float64"
+ },
+ "severe_toxicity": {
+ "dtype": "float64"
+ },
+ "obscene": {
+ "dtype": "float64"
+ },
+ "identity_attack": {
+ "dtype": "float64"
+ },
+ "insult": {
+ "dtype": "float64"
+ },
+ "threat": {
+ "dtype": "float64"
+ },
+ "sexual_explicit": {
+ "dtype": "float64"
+ }
+ }
+ },
+ "message_tree_id": {
+ "dtype": "string"
+ },
+ "tree_state": {
+ "dtype": "string"
+ },
+ "emojis": {
+ "fields": {
+ "name": {
+ "repeated_field": {
+ "dtype": "string"
+ }
+ },
+ "count": {
+ "repeated_field": {
+ "dtype": "int32"
+ }
+ }
+ }
+ },
+ "labels": {
+ "fields": {
+ "name": {
+ "repeated_field": {
+ "dtype": "string"
+ }
+ },
+ "value": {
+ "repeated_field": {
+ "dtype": "float64"
+ }
+ },
+ "count": {
+ "repeated_field": {
+ "dtype": "int32"
+ }
+ }
+ }
+ },
+ "__hfsplit__": {
+ "dtype": "string"
+ },
+ "__rowid__": {
+ "dtype": "string"
+ }
+ }
+ }
+}

data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:138c8efe1e911904c3702c582b892acc8c5616062a35773c31872a8969e2badf
+size 327991072

data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8bcd3f617d324acd7e13d0d0fabd38065012bea40141579e16681bcdfdcaf46
+size 6171232

data/datasets/lilac/open-asssistant-conversations/text/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+ "files": [],
+ "parquet_id": "gte-small(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "gte-small": {
+ "repeated_field": {
+ "fields": {
+ "embedding": {
+ "dtype": "embedding"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "signal_name": "gte-small"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "gte-small"
+ },
+ "enriched_path": [
+ "text"
+ ],
+ "vector_store": "hnsw"
+}

data/datasets/lilac/open-asssistant-conversations/text/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1cda70a8dc3259ff058e5e3ffc24cfbaaafe3fb9ba5c1b836e0757180114e28
+size 5164058

data/datasets/lilac/open-asssistant-conversations/text/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f8944421e23764080d8fde7460d08aa683ebbafc6fad2bd65654ea701ba50ca
+size 2980981

data/datasets/lilac/open-asssistant-conversations/text/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "lang_detection(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "lang_detection": {
+ "dtype": "string",
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/lilac/open-asssistant-conversations/text/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2949cfca1b91bb99c56364fdb47679301b90d1f51bd1963f04fbbcbe093d15c
+size 3486319

data/datasets/lilac/open-asssistant-conversations/text/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "near_dup(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "near_dup": {
+ "fields": {
+ "cluster_id": {
+ "dtype": "uint32",
+ "categorical": true
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/lilac/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c010e414a4379f8c1637c54864c46c7872a7ed0dc26990c5b755581d2073f8b
+size 2953059

data/datasets/lilac/open-asssistant-conversations/text/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "pii(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "pii": {
+ "fields": {
+ "emails": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "ip_addresses": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "secrets": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/lilac/open-asssistant-conversations/text/spacy_ner/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e011c0efb333a2d028e1be33030bf795fc373f27a3c2ce611099081057df2be
+size 5955273

data/datasets/lilac/open-asssistant-conversations/text/spacy_ner/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "spacy_ner(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "spacy_ner": {
+ "repeated_field": {
+ "fields": {
+ "label": {
+ "dtype": "string"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/lilac/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb0f5af1af587a3b083dd7859f9cd4a5cf2943e41396c776db9a2a4f59eb4c9d
+size 3827015

data/datasets/lilac/open-asssistant-conversations/text/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "text_statistics(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "text_statistics": {
+ "fields": {
+ "num_characters": {
+ "dtype": "int32"
+ },
+ "readability": {
+ "dtype": "float32"
+ },
+ "log(type_token_ratio)": {
+ "dtype": "float32"
+ },
+ "frac_non_ascii": {
+ "dtype": "float32",
+ "bins": [
+ [
+ "Low",
+ null,
+ 0.15
+ ],
+ [
+ "Medium",
+ 0.15,
+ 0.3
+ ],
+ [
+ "High",
+ 0.3,
+ null
+ ]
+ ]
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}