Push to HF space

Dockerfile

@@ -25,14 +25,15 @@ RUN python -m pip install lilac[all]
 COPY --chown=user /dist ./dist/
 RUN python -m pip install --find-links=dist --upgrade lilac[all]
 
-COPY --chown=user .env .
-COPY --chown=user .env.demo .
+# Install the huggingface hub, used to download files.
+RUN pip install huggingface_hub
+
 # Copy the README so we can read the datasets from the HuggingFace config.
 COPY --chown=user README.md .
 # Copy the license just in case.
 COPY --chown=user LICENSE .
 
-COPY --chown=user docker_start.sh docker_start.py ./
+COPY --chown=user docker_start.sh ./
 
 # Make a local data directory for non-persistent storage demos.
 RUN mkdir -p ./data

README.md

@@ -6,13 +6,11 @@ colorTo: purple
 sdk: docker
 app_port: 5432
 datasets:
+ - lilacai/lilac-textbook_quality_programming
  - lilacai/lilac-piqa
  - lilacai/lilac-science-qa-derek-thomas
  - lilacai/lilac-enron-emails
- - lilacai/lilac-mmlu_professional_law
- - lilacai/lilac-pile-of-law-r-legaladvice
  - lilacai/lilac-open-asssistant-conversations
- - lilacai/lilac-squad_v2
  - lilacai/lilac-imdb
  - lilacai/lilac-opus100-en-es-validation
  - lilacai/lilac-databricks-dolly-15k-curated-en

data/lilac.yml

@@ -279,277 +279,6 @@ datasets:
  - text
  markdown_paths: []
  preferred_embedding: gte-small
- - namespace: lilac
- name: squad_v2
- tags: []
- source:
- dataset_name: squad_v2
- config_name: null
- split: null
- sample_size: null
- revision: null
- load_from_disk: false
- source_name: huggingface
- embeddings:
- - path: context
- embedding: gte-small
- signals:
- - path: context
- signal:
- threshold: 0.85
- signal_name: near_dup
- - path: context
- signal:
- signal_name: pii
- - path: context
- signal:
- split_by_paragraph: false
- signal_name: lang_detection
- - path: context
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: positive-sentiment
- draft: main
- signal_name: concept_score
- - path: context
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: non-english
- draft: main
- signal_name: concept_score
- - path: context
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: toxicity
- draft: main
- signal_name: concept_score
- - path: context
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: question
- draft: main
- signal_name: concept_score
- - path: context
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: legal-termination
- draft: main
- signal_name: concept_score
- - path: context
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: source-code
- draft: main
- signal_name: concept_score
- - path: context
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: negative-sentiment
- draft: main
- signal_name: concept_score
- - path: context
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: profanity
- draft: main
- signal_name: concept_score
- - path: context
- signal:
- signal_name: text_statistics
- - path: question
- signal:
- threshold: 0.85
- signal_name: near_dup
- - path: question
- signal:
- signal_name: pii
- - path: question
- signal:
- split_by_paragraph: false
- signal_name: lang_detection
- - path: question
- signal:
- signal_name: text_statistics
- - path:
- - answers
- - text
- - '*'
- signal:
- threshold: 0.85
- signal_name: near_dup
- - path:
- - answers
- - text
- - '*'
- signal:
- signal_name: pii
- - path:
- - answers
- - text
- - '*'
- signal:
- split_by_paragraph: false
- signal_name: lang_detection
- - path:
- - answers
- - text
- - '*'
- signal:
- signal_name: text_statistics
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: legal-termination
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: negative-sentiment
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: non-english
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: positive-sentiment
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: profanity
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: question
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: source-code
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: toxicity
- draft: main
- signal_name: concept_score
- - path:
- - answers
- - text
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: legal-termination
- draft: main
- signal_name: concept_score
- - path:
- - answers
- - text
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: negative-sentiment
- draft: main
- signal_name: concept_score
- - path:
- - answers
- - text
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: non-english
- draft: main
- signal_name: concept_score
- - path:
- - answers
- - text
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: positive-sentiment
- draft: main
- signal_name: concept_score
- - path:
- - answers
- - text
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: profanity
- draft: main
- signal_name: concept_score
- - path:
- - answers
- - text
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: question
- draft: main
- signal_name: concept_score
- - path:
- - answers
- - text
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: source-code
- draft: main
- signal_name: concept_score
- - path:
- - answers
- - text
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: toxicity
- draft: main
- signal_name: concept_score
- settings:
- ui:
- media_paths:
- - context
- - question
- - - answers
- - text
- - '*'
- markdown_paths: []
- preferred_embedding: gte-small
  - namespace: lilac
  name: databricks-dolly-15k-curated-en
  tags: []
@@ -1735,319 +1464,28 @@ datasets:
  markdown_paths: []
  preferred_embedding: gte-small
  - namespace: lilac
- name: mmlu_professional_law
+ name: science-qa-derek-thomas
  tags: []
  source:
- dataset_name: cais/mmlu
- config_name: professional_law
+ dataset_name: derek-thomas/ScienceQA
+ config_name: null
  split: null
  sample_size: null
  revision: null
  load_from_disk: false
  source_name: huggingface
  embeddings:
- - path: question
- embedding: gte-small
- - path:
- - choices
- - '*'
+ - path: lecture
  embedding: gte-small
  signals:
- - path: question
+ - path: lecture
  signal:
  threshold: 0.85
  signal_name: near_dup
- - path: question
+ - path: lecture
  signal:
  signal_name: pii
- - path: question
- signal:
- split_by_paragraph: false
- signal_name: lang_detection
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: positive-sentiment
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: non-english
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: toxicity
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: question
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: legal-termination
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: source-code
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: negative-sentiment
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: profanity
- draft: main
- signal_name: concept_score
- - path: question
- signal:
- signal_name: text_statistics
- - path:
- - choices
- - '*'
- signal:
- threshold: 0.85
- signal_name: near_dup
- - path:
- - choices
- - '*'
- signal:
- signal_name: pii
- - path:
- - choices
- - '*'
- signal:
- split_by_paragraph: false
- signal_name: lang_detection
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: positive-sentiment
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: non-english
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: toxicity
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: question
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: legal-termination
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: source-code
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: negative-sentiment
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: negative-sentiment
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: profanity
- draft: main
- signal_name: concept_score
- - path:
- - choices
- - '*'
- signal:
- signal_name: text_statistics
- settings:
- ui:
- media_paths:
- - question
- - - choices
- - '*'
- markdown_paths: []
- preferred_embedding: gte-small
- - namespace: lilac
- name: pile-of-law-r-legaladvice
- tags: []
- source:
- dataset_name: pile-of-law/pile-of-law
- config_name: r_legaladvice
- split: null
- sample_size: null
- revision: null
- load_from_disk: false
- source_name: huggingface
- embeddings:
- - path: text
- embedding: gte-small
- signals:
- - path: text
- signal:
- threshold: 0.85
- signal_name: near_dup
- - path: text
- signal:
- signal_name: pii
- - path: text
- signal:
- split_by_paragraph: false
- signal_name: lang_detection
- - path: text
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: positive-sentiment
- draft: main
- signal_name: concept_score
- - path: text
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: non-english
- draft: main
- signal_name: concept_score
- - path: text
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: toxicity
- draft: main
- signal_name: concept_score
- - path: text
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: question
- draft: main
- signal_name: concept_score
- - path: text
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: legal-termination
- draft: main
- signal_name: concept_score
- - path: text
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: source-code
- draft: main
- signal_name: concept_score
- - path: text
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: negative-sentiment
- draft: main
- signal_name: concept_score
- - path: text
- signal:
- embedding: gte-small
- namespace: lilac
- concept_name: profanity
- draft: main
- signal_name: concept_score
- - path: text
- signal:
- signal_name: text_statistics
- settings:
- ui:
- media_paths:
- - text
- markdown_paths: []
- preferred_embedding: gte-small
- - namespace: lilac
- name: science-qa-derek-thomas
- tags: []
- source:
- dataset_name: derek-thomas/ScienceQA
- config_name: null
- split: null
- sample_size: null
- revision: null
- load_from_disk: false
- source_name: huggingface
- embeddings:
- - path: lecture
- embedding: gte-small
- signals:
- - path: lecture
- signal:
- threshold: 0.85
- signal_name: near_dup
- - path: lecture
- signal:
- signal_name: pii
- - path: lecture
+ - path: lecture
  signal:
  split_by_paragraph: false
  signal_name: lang_detection
@@ -2297,5 +1735,39 @@ datasets:
  - overview
  markdown_paths: []
  preferred_embedding: gte-small
+ - namespace: lilac
+ name: textbook_quality_programming
+ tags: []
+ source:
+ dataset_name: vikp/textbook_quality_programming
+ config_name: null
+ split: null
+ sample_size: null
+ revision: null
+ load_from_disk: false
+ source_name: huggingface
+ embeddings:
+ - path:
+ - outline
+ - '*'
+ embedding: gte-small
+ - path:
+ - concepts
+ - '*'
+ embedding: gte-small
+ - path: markdown
+ embedding: gte-small
+ signals: []
+ settings:
+ ui:
+ media_paths:
+ - - outline
+ - '*'
+ - - concepts
+ - '*'
+ - markdown
+ markdown_paths:
+ - markdown
+ preferred_embedding: gte-small
 signals: []
 concept_model_cache_embeddings: []

docker_start.sh

@@ -3,7 +3,7 @@
 # Fail if any of the commands below fail.
 set -e
 
-python docker_start.py
+lilac hf-docker-start
 gunicorn lilac.server:app \
  --bind 0.0.0.0:5432 \
  --preload -k uvicorn.workers.UvicornWorker \