Upload folder using huggingface_hub

.gitattributes

@@ -0,0 +1,4 @@
+data/datasets/lilac/pile-of-law-constitutions/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/pile-of-law-constitutions/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/pile-of-law-constitutions/config.yml

@@ -0,0 +1,23 @@
+embeddings:
+- embedding: gte-small
+ path: text
+name: pile-of-law-constitutions
+namespace: lilac
+settings:
+ preferred_embedding: gte-small
+ ui:
+ media_paths:
+ - text
+signals:
+- path: text
+ signal:
+ signal_name: near_dup
+- path: text
+ signal:
+ signal_name: pii
+source:
+ config_name: constitutions
+ dataset_name: pile-of-law/pile-of-law
+ source_name: huggingface
+tags:
+- legal

data/datasets/lilac/pile-of-law-constitutions/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95d5ed1a03bb7960e4404a2824e56edc196bc868f2653083d76bd15785354d9b
+size 11644007

data/datasets/lilac/pile-of-law-constitutions/manifest.json

@@ -0,0 +1,27 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "data_schema": {
+ "fields": {
+ "text": {
+ "dtype": "string"
+ },
+ "created_timestamp": {
+ "dtype": "string"
+ },
+ "downloaded_timestamp": {
+ "dtype": "string"
+ },
+ "url": {
+ "dtype": "string"
+ },
+ "__hfsplit__": {
+ "dtype": "string"
+ },
+ "__rowid__": {
+ "dtype": "string"
+ }
+ }
+ }
+}

data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0191d5390ec25b1a5df6558e31eebfb6816ba4b0d056782a1734be53f38ec25
+size 165963072

data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0391b6a70d478587e8e4880da808b3964711040244e72030392f7c1cdac7ad4
+size 1832634

data/datasets/lilac/pile-of-law-constitutions/text/gte-small/signal_manifest.json

@@ -0,0 +1,35 @@
+{
+ "files": [],
+ "parquet_id": "gte-small(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "gte-small": {
+ "repeated_field": {
+ "fields": {
+ "embedding": {
+ "dtype": "embedding"
+ }
+ },
+ "dtype": "string_span"
+ },
+ "signal": {
+ "signal_name": "gte-small"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "gte-small"
+ },
+ "enriched_path": [
+ "text"
+ ],
+ "vector_store": "hnsw"
+}

data/datasets/lilac/pile-of-law-constitutions/text/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8daf511d5157878cacab7058e516638d8c3c354434cb2792f9dd2a83637b21b5
+size 1029851

data/datasets/lilac/pile-of-law-constitutions/text/near_dup/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "near_dup(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "near_dup": {
+ "fields": {
+ "cluster_id": {
+ "dtype": "uint32",
+ "categorical": true
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "threshold": 0.85,
+ "signal_name": "near_dup"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}

data/datasets/lilac/pile-of-law-constitutions/text/pii/signal_manifest.json

@@ -0,0 +1,45 @@
+{
+ "files": [
+ "data-00000-of-00001.parquet"
+ ],
+ "parquet_id": "pii(text)",
+ "data_schema": {
+ "fields": {
+ "__rowid__": {
+ "dtype": "string"
+ },
+ "text": {
+ "fields": {
+ "pii": {
+ "fields": {
+ "emails": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "ip_addresses": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ },
+ "secrets": {
+ "repeated_field": {
+ "dtype": "string_span"
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ }
+ }
+ }
+ }
+ }
+ },
+ "signal": {
+ "signal_name": "pii"
+ },
+ "enriched_path": [
+ "text"
+ ]
+}