Upload folder using huggingface_hub

.gitattributes

@@ -48,3 +48,22 @@ data/datasets/lilac/open-asssistant-conversations/text/near_dup/data-00000-of-00
 data/datasets/lilac/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/open-asssistant-conversations/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/answers/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/answers/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/answers/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/answers/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/answers/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/context/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/context/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/context/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/context/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/context/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/context/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/context/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/context/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/question/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/squad_v2/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/squad_v2/answers/text/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c03103ba132a7209461f86bd1045431d06db431930344e4bdf97236347cc2164
+size 4738120

data/datasets/lilac/squad_v2/answers/text/lang_detection/signal_manifest.json

@@ -0,0 +1,39 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "lang_detection(answers.text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "answers": {
+ "fields": {
+ "text": {
+ "repeated_field": {
+ "fields": {
+ "lang_detection": {
+ "dtype": "string",
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ },
+ "enriched_path": [
+ "answers",
+ "text",
+ "*"
+ ]
+}

data/datasets/lilac/squad_v2/answers/text/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf4ae7259d126104da2aea0e1fad0c7cd83033f7774f0d44a2436f7c891fde34
+size 5224344

data/datasets/lilac/squad_v2/answers/text/near_dup/signal_manifest.json

@@ -0,0 +1,44 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "near_dup(answers.text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "answers": {
+ "fields": {
+ "text": {
+ "repeated_field": {
+ "fields": {
+ "near_dup": {
+ "fields": {
+ "cluster_id": {
+ "dtype": "uint32",
+ "categorical": true
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ },
+ "enriched_path": [
+ "answers",
+ "text",
+ "*"
+ ]
+}

data/datasets/lilac/squad_v2/answers/text/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adf2c35877ae9957a987049c40a9a1b2edbe4b2d93b1da86bfeb739fae240040
+size 4841393

data/datasets/lilac/squad_v2/answers/text/pii/signal_manifest.json

@@ -0,0 +1,53 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "pii(answers.text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "answers": {
+ "fields": {
+ "text": {
+ "repeated_field": {
+ "fields": {
+ "pii": {
+ "fields": {
+ "emails": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "ip_addresses": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "secrets": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ },
+ "enriched_path": [
+ "answers",
+ "text",
+ "*"
+ ]
+}

data/datasets/lilac/squad_v2/answers/text/spacy_ner/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86cfdb80cf22a545cd01d557ae3396e942fa9679990c43457d110a6dac6a2d78
+size 5041580

data/datasets/lilac/squad_v2/answers/text/spacy_ner/signal_manifest.json

@@ -0,0 +1,46 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "spacy_ner(answers.text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "answers": {
+ "fields": {
+ "text": {
+ "repeated_field": {
+ "fields": {
+ "spacy_ner": {
+ "repeated_field": {
+ "fields": {
+ "label": {
+ "dtype": "string"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ },
+ "enriched_path": [
+ "answers",
+ "text",
+ "*"
+ ]
+}

data/datasets/lilac/squad_v2/answers/text/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16b5ddbc51455341a26121c5427bd0f32639515dad34d77561402df81d8ab903
+size 5100206

data/datasets/lilac/squad_v2/answers/text/text_statistics/signal_manifest.json

@@ -0,0 +1,67 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "text_statistics(answers.text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "answers": {
+ "fields": {
+ "text": {
+ "repeated_field": {
+ "fields": {
+ "text_statistics": {
+ "fields": {
+ "num_characters": {
+ "dtype": "int32"
+ },
+ "readability": {
+ "dtype": "float32"
+ },
+ "log(type_token_ratio)": {
+ "dtype": "float32"
+ },
+ "frac_non_ascii": {
+ "dtype": "float32",
+ "bins": [
+ [
+ "Low",
+ null,
+ 0.15
+ ],
+ [
+ "Medium",
+ 0.15,
+ 0.3
+ ],
+ [
+ "High",
+ 0.3,
+ null
+ ]
+ ]
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ },
+ "enriched_path": [
+ "answers",
+ "text",
+ "*"
+ ]
+}

data/datasets/lilac/squad_v2/config.yml

@@ -0,0 +1,80 @@
+embeddings:
+- embedding: gte-small
+ path: context
+name: squad_v2
+namespace: local
+settings:
+ preferred_embedding: gte-small
+ ui:
+ media_paths:
+ - context
+ - question
+ - - answers
+ - text
+ - '*'
+signals:
+- path: context
+ signal:
+ signal_name: text_statistics
+- path: context
+ signal:
+ signal_name: pii
+- path: context
+ signal:
+ signal_name: near_dup
+- path: question
+ signal:
+ signal_name: spacy_ner
+- path: question
+ signal:
+ signal_name: pii
+- path:
+ - answers
+ - text
+ - '*'
+ signal:
+ signal_name: pii
+- path:
+ - answers
+ - text
+ - '*'
+ signal:
+ signal_name: spacy_ner
+- path:
+ - answers
+ - text
+ - '*'
+ signal:
+ signal_name: near_dup
+- path: context
+ signal:
+ signal_name: lang_detection
+- path:
+ - answers
+ - text
+ - '*'
+ signal:
+ signal_name: lang_detection
+- path: question
+ signal:
+ signal_name: near_dup
+- path: question
+ signal:
+ signal_name: lang_detection
+- path:
+ - answers
+ - text
+ - '*'
+ signal:
+ signal_name: text_statistics
+- path: question
+ signal:
+ signal_name: text_statistics
+- path: context
+ signal:
+ signal_name: spacy_ner
+source:
+ dataset_name: squad_v2
+ source_name: huggingface
+tags:
+- machine-learning

data/datasets/lilac/squad_v2/context/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e24a3d0200e46bb221dacc8066ccc85033ff0378721338cfd60612f130e034d1
+size 601394376

data/datasets/lilac/squad_v2/context/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc41116b96f4e1fa547697ce62afe0fe7aba054a8d694b308e1e0270474801da
+size 10694495

data/datasets/lilac/squad_v2/context/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+ "files": [],
+ "parquet_id": "gte-small(context)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "context": {
+ "fields": {
+ "gte-small": {
+ "repeated_field": {
+ "fields": {
+ "embedding": {
+ "dtype": "embedding"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "signal_name": "gte-small"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "gte-small"
+ },
+ "enriched_path": [
+ "context"
+ ],
+ "vector_store": "hnsw"
+}

data/datasets/lilac/squad_v2/context/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d69a524ee48c0c218eeb901ae265ae74b12511fee17fe31ae1627c0122e25f04
+size 8815907

data/datasets/lilac/squad_v2/context/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd10704958387059935d9e22f0415677a6daf3105105af8314314ce3c3114274
+size 4682949

data/datasets/lilac/squad_v2/context/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "lang_detection(context)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "context": {
+ "fields": {
+ "lang_detection": {
+ "dtype": "string",
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ },
+ "enriched_path": [
+ "context"
+ ]
+}

data/datasets/lilac/squad_v2/context/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0d3a359a05aa7c073900e4973569f808afa26c7bf0328c31e553efcc14bea90
+size 4962702

data/datasets/lilac/squad_v2/context/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "near_dup(context)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "context": {
+ "fields": {
+ "near_dup": {
+ "fields": {
+ "cluster_id": {
+ "dtype": "uint32",
+ "categorical": true
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ },
+ "enriched_path": [
+ "context"
+ ]
+}

data/datasets/lilac/squad_v2/context/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0265330a9d7ff27498f4f0e9ddce89a027203d11941d6bc8f8d4334872346d9c
+size 4685328

data/datasets/lilac/squad_v2/context/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "pii(context)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "context": {
+ "fields": {
+ "pii": {
+ "fields": {
+ "emails": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "ip_addresses": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "secrets": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ },
+ "enriched_path": [
+ "context"
+ ]
+}

data/datasets/lilac/squad_v2/context/spacy_ner/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffe9e61a5449146e9b7cf725b80f46277b359a1aca1d04fe15c5bae4e9f286f1
+size 9241062

data/datasets/lilac/squad_v2/context/spacy_ner/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "spacy_ner(context)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "context": {
+ "fields": {
+ "spacy_ner": {
+ "repeated_field": {
+ "fields": {
+ "label": {
+ "dtype": "string"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ },
+ "enriched_path": [
+ "context"
+ ]
+}

data/datasets/lilac/squad_v2/context/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1915970f9199dc82019cbb6089c85df3ddfd189848e0f34f549e34b617cd0f8
+size 5165481

data/datasets/lilac/squad_v2/context/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "text_statistics(context)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "context": {
+ "fields": {
+ "text_statistics": {
+ "fields": {
+ "num_characters": {
+ "dtype": "int32"
+ },
+ "readability": {
+ "dtype": "float32"
+ },
+ "log(type_token_ratio)": {
+ "dtype": "float32"
+ },
+ "frac_non_ascii": {
+ "dtype": "float32",
+ "bins": [
+ [
+ "Low",
+ null,
+ 0.15
+ ],
+ [
+ "Medium",
+ 0.15,
+ 0.3
+ ],
+ [
+ "High",
+ 0.3,
+ null
+ ]
+ ]
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ },
+ "enriched_path": [
+ "context"
+ ]
+}

data/datasets/lilac/squad_v2/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9f54db85b8bacd3ea30ecf70410441e300c783e621767c1d9746d6474852ceb
+size 27086838

data/datasets/lilac/squad_v2/manifest.json

@@ -0,0 +1,41 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "data_schema": {
+ "fields": {
+ "id": {
+ "dtype": "string"
+ },
+ "title": {
+ "dtype": "string"
+ },
+ "context": {
+ "dtype": "string"
+ },
+ "question": {
+ "dtype": "string"
+ },
+ "answers": {
+ "fields": {
+ "text": {
+ "repeated_field": {
+ "dtype": "string"
+ }
+ },
+ "answer_start": {
+ "repeated_field": {
+ "dtype": "int32"
+ }
+ }
+ }
+ },
+ "__hfsplit__": {
+ "dtype": "string"
+ },
+ "__rowid__": {
+ "dtype": "string"
+ }
+ }
+ }
+}

data/datasets/lilac/squad_v2/question/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06d32aa3096865a6236fd620a16499876c919b245e26fe9a2809b3c02eebc13d
+size 4694280

data/datasets/lilac/squad_v2/question/lang_detection/signal_manifest.json

@@ -0,0 +1,31 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "lang_detection(question)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "question": {
+ "fields": {
+ "lang_detection": {
+ "dtype": "string",
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "split_by_paragraph": false,
+ "signal_name": "lang_detection"
+ },
+ "enriched_path": [
+ "question"
+ ]
+}

data/datasets/lilac/squad_v2/question/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b7e1d521750d16c37c70c378306ef22916e2d9715a565f1127d9e3626c966d4
+size 5571030

data/datasets/lilac/squad_v2/question/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "near_dup(question)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "question": {
+ "fields": {
+ "near_dup": {
+ "fields": {
+ "cluster_id": {
+ "dtype": "uint32",
+ "categorical": true
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ },
+ "enriched_path": [
+ "question"
+ ]
+}

data/datasets/lilac/squad_v2/question/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a72f4f32331de183cfe67be224ba473ec83ba8f855dafab97371580684718e4f
+size 4685523

data/datasets/lilac/squad_v2/question/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "pii(question)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "question": {
+ "fields": {
+ "pii": {
+ "fields": {
+ "emails": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "ip_addresses": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "secrets": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ },
+ "enriched_path": [
+ "question"
+ ]
+}

data/datasets/lilac/squad_v2/question/spacy_ner/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca09ac95100482b232d2b3c7e9a08490ecb056d10b9e91a23170df6669482a3b
+size 5286963

data/datasets/lilac/squad_v2/question/spacy_ner/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "spacy_ner(question)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "question": {
+ "fields": {
+ "spacy_ner": {
+ "repeated_field": {
+ "fields": {
+ "label": {
+ "dtype": "string"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "model": "en_core_web_sm",
+ "signal_name": "spacy_ner"
+ },
+ "enriched_path": [
+ "question"
+ ]
+}

data/datasets/lilac/squad_v2/question/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36304efa2147c8737c1c4da192837aa855504ecfe9eb6f14d267c501bcaaa246
+size 5104750

data/datasets/lilac/squad_v2/question/text_statistics/signal_manifest.json

@@ -0,0 +1,59 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "text_statistics(question)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "question": {
+ "fields": {
+ "text_statistics": {
+ "fields": {
+ "num_characters": {
+ "dtype": "int32"
+ },
+ "readability": {
+ "dtype": "float32"
+ },
+ "log(type_token_ratio)": {
+ "dtype": "float32"
+ },
+ "frac_non_ascii": {
+ "dtype": "float32",
+ "bins": [
+ [
+ "Low",
+ null,
+ 0.15
+ ],
+ [
+ "Medium",
+ 0.15,
+ 0.3
+ ],
+ [
+ "High",
+ 0.3,
+ null
+ ]
+ ]
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "text_statistics"
+ },
+ "enriched_path": [
+ "question"
+ ]
+}