Push

.gitattributes

@@ -1,87 +0,0 @@
-data/datasets/lilac/pile-of-law-constitutions/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-constitutions/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-atticus-contracts/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -21,4 +21,4 @@ COPY /lilac ./lilac/
 
 COPY docker_start.sh docker_start.py ./
 
-CMD ./docker_start.sh
+CMD ["bash", "docker_start.sh"]

data/datasets/lilac/imdb/config.yml

@@ -1,31 +0,0 @@
-embeddings:
-- embedding: gte-small
- path: text
-name: imdb
-namespace: local
-settings:
- preferred_embedding: gte-small
- ui:
- media_paths:
- - text
-signals:
-- path: text
- signal:
- signal_name: near_dup
-- path: text
- signal:
- signal_name: text_statistics
-- path: text
- signal:
- signal_name: lang_detection
-- path: text
- signal:
- signal_name: spacy_ner
-- path: text
- signal:
- signal_name: pii
-source:
- dataset_name: imdb
- source_name: huggingface
-tags:
-- machine-learning

data/datasets/lilac/imdb/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6fe90be23f86ca1e73b8a77a235344db822601f794a5643dca9d0d07c49ce3d8
-size 86160450

data/datasets/lilac/imdb/manifest.json

@@ -1,21 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "data_schema": {
- "fields": {
- "text": {
- "dtype": "string"
- },
- "label": {
- "dtype": "string"
- },
- "__hfsplit__": {
- "dtype": "string"
- },
- "__rowid__": {
- "dtype": "string"
- }
- }
- }
-}

data/datasets/lilac/imdb/text/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7b8fc440b947f068966bc68c83ce4b5502b98e3aab928d58c295dc8a7b7b016c
-size 691432396

data/datasets/lilac/imdb/text/gte-small/hnsw.lookup.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d3ee2426c761b50025e02f2388fa65c51eb9705d6fd65f95d6502650214b8472
-size 10390867

data/datasets/lilac/imdb/text/gte-small/signal_manifest.json

@@ -1,35 +0,0 @@
-{
- "files": [],
- "parquet_id": "gte-small(text)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "text": {
- "fields": {
- "gte-small": {
- "repeated_field": {
- "fields": {
- "embedding": {
- "dtype": "embedding"
- }
- },
- "dtype": "string_span"
- },
- "signal": {
- "signal_name": "gte-small"
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "gte-small"
- },
- "enriched_path": [
- "text"
- ],
- "vector_store": "hnsw"
-}

data/datasets/lilac/imdb/text/gte-small/spans.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:16417449057fc4304e098e037cb2b3e9a693570768a68bdca457d452adaee130
-size 7476546

data/datasets/lilac/imdb/text/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:adba6df16ec47b68625618b01a0b8d2cc65de572acb20eed561128b037fcdfd7
-size 3309315

data/datasets/lilac/imdb/text/lang_detection/signal_manifest.json

@@ -1,31 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "lang_detection(text)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "text": {
- "fields": {
- "lang_detection": {
- "dtype": "string",
- "signal": {
- "split_by_paragraph": false,
- "signal_name": "lang_detection"
- }
- }
- }
- }
- }
- },
- "signal": {
- "split_by_paragraph": false,
- "signal_name": "lang_detection"
- },
- "enriched_path": [
- "text"
- ]
-}

data/datasets/lilac/imdb/text/near_dup/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0c75a5f0d12b8b02671e99a2dd313d292610c9264f2302ad1877959430420079
-size 3915752

data/datasets/lilac/imdb/text/near_dup/signal_manifest.json

@@ -1,36 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "near_dup(text)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "text": {
- "fields": {
- "near_dup": {
- "fields": {
- "cluster_id": {
- "dtype": "uint32",
- "categorical": true
- }
- },
- "signal": {
- "threshold": 0.85,
- "signal_name": "near_dup"
- }
- }
- }
- }
- }
- },
- "signal": {
- "threshold": 0.85,
- "signal_name": "near_dup"
- },
- "enriched_path": [
- "text"
- ]
-}

data/datasets/lilac/imdb/text/pii/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2dfe71845bfd419a2a309b132456341bf317d63e8ccf8bc100835f1a20c81c5b
-size 3313701

data/datasets/lilac/imdb/text/pii/signal_manifest.json

@@ -1,45 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "pii(text)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "text": {
- "fields": {
- "pii": {
- "fields": {
- "emails": {
- "repeated_field": {
- "dtype": "string_span"
- }
- },
- "ip_addresses": {
- "repeated_field": {
- "dtype": "string_span"
- }
- },
- "secrets": {
- "repeated_field": {
- "dtype": "string_span"
- }
- }
- },
- "signal": {
- "signal_name": "pii"
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "pii"
- },
- "enriched_path": [
- "text"
- ]
-}

data/datasets/lilac/imdb/text/spacy_ner/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:be39eb80c981da64a2ec5f7fe42c43ffb390b887a220e278a58cdde04e7824ef
-size 8479478

data/datasets/lilac/imdb/text/spacy_ner/signal_manifest.json

@@ -1,38 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "spacy_ner(text)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "text": {
- "fields": {
- "spacy_ner": {
- "repeated_field": {
- "fields": {
- "label": {
- "dtype": "string"
- }
- },
- "dtype": "string_span"
- },
- "signal": {
- "model": "en_core_web_sm",
- "signal_name": "spacy_ner"
- }
- }
- }
- }
- }
- },
- "signal": {
- "model": "en_core_web_sm",
- "signal_name": "spacy_ner"
- },
- "enriched_path": [
- "text"
- ]
-}

data/datasets/lilac/imdb/text/text_statistics/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4919817d0c520fc5aacc9721f01a0e29fb4d2c89b6698865c9680c81c44ce920
-size 4403809

data/datasets/lilac/imdb/text/text_statistics/signal_manifest.json

@@ -1,59 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "text_statistics(text)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "text": {
- "fields": {
- "text_statistics": {
- "fields": {
- "num_characters": {
- "dtype": "int32"
- },
- "readability": {
- "dtype": "float32"
- },
- "log(type_token_ratio)": {
- "dtype": "float32"
- },
- "frac_non_ascii": {
- "dtype": "float32",
- "bins": [
- [
- "Low",
- null,
- 0.15
- ],
- [
- "Medium",
- 0.15,
- 0.3
- ],
- [
- "High",
- 0.3,
- null
- ]
- ]
- }
- },
- "signal": {
- "signal_name": "text_statistics"
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "text_statistics"
- },
- "enriched_path": [
- "text"
- ]
-}

data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:df9d6e2f5df4b8693544f31ca78a9d1936a4caf47acc2babeb1cb766131b7636
-size 684360968

data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.lookup.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2081ce5d760026fe341e0553cd9e40747ca902e4e7edb851cb747f350f19bb0d
-size 11174465

data/datasets/lilac/mmlu_professional_law/choices/gte-small/signal_manifest.json

@@ -1,38 +0,0 @@
-{
- "files": [],
- "parquet_id": "gte-small(choices)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "choices": {
- "repeated_field": {
- "fields": {
- "gte-small": {
- "repeated_field": {
- "fields": {
- "embedding": {
- "dtype": "embedding"
- }
- },
- "dtype": "string_span"
- },
- "signal": {
- "signal_name": "gte-small"
- }
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "gte-small"
- },
- "enriched_path": [
- "choices",
- "*"
- ],
- "vector_store": "hnsw"
-}

data/datasets/lilac/mmlu_professional_law/choices/gte-small/spans.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:02fb1662da21f33ea1429a0f9adf1301185da46f642a722717fe7c523314fa57
-size 11173475

data/datasets/lilac/mmlu_professional_law/choices/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:968d4f87c7b51b995d9e3a96423a06b91984e5ee4a47062cd53fe87cca5cafbe
-size 3469413

data/datasets/lilac/mmlu_professional_law/choices/lang_detection/signal_manifest.json

@@ -1,34 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "lang_detection(choices)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "choices": {
- "repeated_field": {
- "fields": {
- "lang_detection": {
- "dtype": "string",
- "signal": {
- "split_by_paragraph": false,
- "signal_name": "lang_detection"
- }
- }
- }
- }
- }
- }
- },
- "signal": {
- "split_by_paragraph": false,
- "signal_name": "lang_detection"
- },
- "enriched_path": [
- "choices",
- "*"
- ]
-}

data/datasets/lilac/mmlu_professional_law/choices/near_dup/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:316f3be499fcbb960bc1e83a201838ca0b3047a71d8e1c302fe4e0d833a3bf90
-size 5544176

data/datasets/lilac/mmlu_professional_law/choices/near_dup/signal_manifest.json

@@ -1,39 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "near_dup(choices)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "choices": {
- "repeated_field": {
- "fields": {
- "near_dup": {
- "fields": {
- "cluster_id": {
- "dtype": "uint32",
- "categorical": true
- }
- },
- "signal": {
- "threshold": 0.85,
- "signal_name": "near_dup"
- }
- }
- }
- }
- }
- }
- },
- "signal": {
- "threshold": 0.85,
- "signal_name": "near_dup"
- },
- "enriched_path": [
- "choices",
- "*"
- ]
-}

data/datasets/lilac/mmlu_professional_law/choices/pii/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7cb41d4e9d0d82bd824abfa733d5be3a599e011098c5d41ebadeb1166a15f722
-size 3393096

data/datasets/lilac/mmlu_professional_law/choices/pii/signal_manifest.json

@@ -1,48 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "pii(choices)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "choices": {
- "repeated_field": {
- "fields": {
- "pii": {
- "fields": {
- "emails": {
- "repeated_field": {
- "dtype": "string_span"
- }
- },
- "ip_addresses": {
- "repeated_field": {
- "dtype": "string_span"
- }
- },
- "secrets": {
- "repeated_field": {
- "dtype": "string_span"
- }
- }
- },
- "signal": {
- "signal_name": "pii"
- }
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "pii"
- },
- "enriched_path": [
- "choices",
- "*"
- ]
-}

data/datasets/lilac/mmlu_professional_law/choices/spacy_ner/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4b1255490f17c64f88b8b332c7df30060df612b9de11b17aaf6f70234c363e1e
-size 4080744

data/datasets/lilac/mmlu_professional_law/choices/spacy_ner/signal_manifest.json

@@ -1,41 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "spacy_ner(choices)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "choices": {
- "repeated_field": {
- "fields": {
- "spacy_ner": {
- "repeated_field": {
- "fields": {
- "label": {
- "dtype": "string"
- }
- },
- "dtype": "string_span"
- },
- "signal": {
- "model": "en_core_web_sm",
- "signal_name": "spacy_ner"
- }
- }
- }
- }
- }
- }
- },
- "signal": {
- "model": "en_core_web_sm",
- "signal_name": "spacy_ner"
- },
- "enriched_path": [
- "choices",
- "*"
- ]
-}

data/datasets/lilac/mmlu_professional_law/choices/text_statistics/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bc00a68e0f835e25b214d90e7e48251b39d748f1e836af713440cd0ea2517ead
-size 4634821

data/datasets/lilac/mmlu_professional_law/choices/text_statistics/signal_manifest.json

@@ -1,62 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "text_statistics(choices)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "choices": {
- "repeated_field": {
- "fields": {
- "text_statistics": {
- "fields": {
- "num_characters": {
- "dtype": "int32"
- },
- "readability": {
- "dtype": "float32"
- },
- "log(type_token_ratio)": {
- "dtype": "float32"
- },
- "frac_non_ascii": {
- "dtype": "float32",
- "bins": [
- [
- "Low",
- null,
- 0.15
- ],
- [
- "Medium",
- 0.15,
- 0.3
- ],
- [
- "High",
- 0.3,
- null
- ]
- ]
- }
- },
- "signal": {
- "signal_name": "text_statistics"
- }
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "text_statistics"
- },
- "enriched_path": [
- "choices",
- "*"
- ]
-}

data/datasets/lilac/mmlu_professional_law/config.yml

@@ -1,63 +0,0 @@
-embeddings:
-- embedding: gte-small
- path:
- - choices
- - '*'
-- embedding: gte-small
- path: question
-name: mmlu_professional_law
-namespace: local
-settings:
- preferred_embedding: gte-small
- ui:
- media_paths:
- - question
- - - choices
- - '*'
-signals:
-- path: question
- signal:
- signal_name: text_statistics
-- path: question
- signal:
- signal_name: pii
-- path: question
- signal:
- signal_name: near_dup
-- path:
- - choices
- - '*'
- signal:
- signal_name: text_statistics
-- path:
- - choices
- - '*'
- signal:
- signal_name: spacy_ner
-- path: question
- signal:
- signal_name: lang_detection
-- path:
- - choices
- - '*'
- signal:
- signal_name: near_dup
-- path:
- - choices
- - '*'
- signal:
- signal_name: pii
-- path:
- - choices
- - '*'
- signal:
- signal_name: lang_detection
-- path: question
- signal:
- signal_name: spacy_ner
-source:
- config_name: professional_law
- dataset_name: cais/mmlu
- source_name: huggingface
-tags:
-- legal

data/datasets/lilac/mmlu_professional_law/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:65cd2771cf0bb88dbed9ad66ceaff472115f07c9dfea866c7e3f65b68392e745
-size 50699938

data/datasets/lilac/mmlu_professional_law/manifest.json

@@ -1,26 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "data_schema": {
- "fields": {
- "question": {
- "dtype": "string"
- },
- "choices": {
- "repeated_field": {
- "dtype": "string"
- }
- },
- "answer": {
- "dtype": "string"
- },
- "__hfsplit__": {
- "dtype": "string"
- },
- "__rowid__": {
- "dtype": "string"
- }
- }
- }
-}

data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3b02300405fccc3011294e15ee869933dd81578173435defbcb19e3b40a65e93
-size 771802212

data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.lookup.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f72169740d80ee2b2ea66589d7ebcc58c83381978a4640a27510c416a02bf6c7
-size 11296648

data/datasets/lilac/mmlu_professional_law/question/gte-small/signal_manifest.json

@@ -1,35 +0,0 @@
-{
- "files": [],
- "parquet_id": "gte-small(question)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "question": {
- "fields": {
- "gte-small": {
- "repeated_field": {
- "fields": {
- "embedding": {
- "dtype": "embedding"
- }
- },
- "dtype": "string_span"
- },
- "signal": {
- "signal_name": "gte-small"
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "gte-small"
- },
- "enriched_path": [
- "question"
- ],
- "vector_store": "hnsw"
-}

data/datasets/lilac/mmlu_professional_law/question/gte-small/spans.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9b51cad455e94b167bc9cf130c262ed1b143a8f386c7074a61983e01cd93d277
-size 7911602

data/datasets/lilac/mmlu_professional_law/question/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bf6cf8cdc246ce4406599aec8782d3be02f2585f1fbad74173faf0ffcb453a49
-size 3361922

data/datasets/lilac/mmlu_professional_law/question/lang_detection/signal_manifest.json

@@ -1,31 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "lang_detection(question)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "question": {
- "fields": {
- "lang_detection": {
- "dtype": "string",
- "signal": {
- "split_by_paragraph": false,
- "signal_name": "lang_detection"
- }
- }
- }
- }
- }
- },
- "signal": {
- "split_by_paragraph": false,
- "signal_name": "lang_detection"
- },
- "enriched_path": [
- "question"
- ]
-}

data/datasets/lilac/mmlu_professional_law/question/near_dup/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2c4139f699d1a248cf5378c442ef6f17970913394d5d0c79bd7c6e6801ab548a
-size 3697516

data/datasets/lilac/mmlu_professional_law/question/near_dup/signal_manifest.json

@@ -1,36 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "near_dup(question)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "question": {
- "fields": {
- "near_dup": {
- "fields": {
- "cluster_id": {
- "dtype": "uint32",
- "categorical": true
- }
- },
- "signal": {
- "threshold": 0.85,
- "signal_name": "near_dup"
- }
- }
- }
- }
- }
- },
- "signal": {
- "threshold": 0.85,
- "signal_name": "near_dup"
- },
- "enriched_path": [
- "question"
- ]
-}

data/datasets/lilac/mmlu_professional_law/question/pii/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2735c4a2c5d40973652d369140533af74425db6dd753f8a25850d4efeee4928e
-size 3369080

data/datasets/lilac/mmlu_professional_law/question/pii/signal_manifest.json

@@ -1,45 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "pii(question)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "question": {
- "fields": {
- "pii": {
- "fields": {
- "emails": {
- "repeated_field": {
- "dtype": "string_span"
- }
- },
- "ip_addresses": {
- "repeated_field": {
- "dtype": "string_span"
- }
- },
- "secrets": {
- "repeated_field": {
- "dtype": "string_span"
- }
- }
- },
- "signal": {
- "signal_name": "pii"
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "pii"
- },
- "enriched_path": [
- "question"
- ]
-}

data/datasets/lilac/mmlu_professional_law/question/spacy_ner/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e775b663f9a3b7c7ebdd31f9a860254dec31c18aa46c5a61820050d0556cbb0f
-size 9105982

data/datasets/lilac/mmlu_professional_law/question/spacy_ner/signal_manifest.json

@@ -1,38 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "spacy_ner(question)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "question": {
- "fields": {
- "spacy_ner": {
- "repeated_field": {
- "fields": {
- "label": {
- "dtype": "string"
- }
- },
- "dtype": "string_span"
- },
- "signal": {
- "model": "en_core_web_sm",
- "signal_name": "spacy_ner"
- }
- }
- }
- }
- }
- },
- "signal": {
- "model": "en_core_web_sm",
- "signal_name": "spacy_ner"
- },
- "enriched_path": [
- "question"
- ]
-}

data/datasets/lilac/mmlu_professional_law/question/text_statistics/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:995b3ac42907ea244d9cb04c68a4715af8ddb7d72dcced056bc58dc9a9f05e7e
-size 4389031

data/datasets/lilac/mmlu_professional_law/question/text_statistics/signal_manifest.json

@@ -1,59 +0,0 @@
-{
- "files": [
- "data-00000-of-00001.parquet"
- ],
- "parquet_id": "text_statistics(question)",
- "data_schema": {
- "fields": {
- "__rowid__": {
- "dtype": "string"
- },
- "question": {
- "fields": {
- "text_statistics": {
- "fields": {
- "num_characters": {
- "dtype": "int32"
- },
- "readability": {
- "dtype": "float32"
- },
- "log(type_token_ratio)": {
- "dtype": "float32"
- },
- "frac_non_ascii": {
- "dtype": "float32",
- "bins": [
- [
- "Low",
- null,
- 0.15
- ],
- [
- "Medium",
- 0.15,
- 0.3
- ],
- [
- "High",
- 0.3,
- null
- ]
- ]
- }
- },
- "signal": {
- "signal_name": "text_statistics"
- }
- }
- }
- }
- }
- },
- "signal": {
- "signal_name": "text_statistics"
- },
- "enriched_path": [
- "question"
- ]
-}