add prompt

pyproject.toml

@@ -12,7 +12,10 @@ dependencies = [
     "torch>=2.4.1",
     "accelerate>=0.34.0",
     "pillow>=10.4.0",
+    "setuptools",
     "flash-attn>=2.6.3",
+    "torchvision>=0.19.1",
+    "pdf2image>=1.17.0",
 ]
 
 [tool.uv]

src/planning_ai/phi.py

@@ -1,4 +1,6 @@
-import requests
+from pathlib import Path
+
+from pdf2image import convert_from_path
 from PIL import Image
 from transformers import AutoModelForCausalLM, AutoProcessor
 
@@ -14,31 +16,43 @@ model = AutoModelForCausalLM.from_pretrained(
 )
 
 # for best performance, use num_crops=4 for multi-frame, num_crops=16 for single-frame.
-processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True, num_crops=4)
+processor = AutoProcessor.from_pretrained(
+    model_id, trust_remote_code=True, num_crops=16
+)
 
 images = []
 placeholder = ""
-
-# Note: if OOM, you might consider reduce number of frames in this example.
-path = "./data/raw/2024-09-05_10-46.png"
-images.append(Image.open(requests.get(path, stream=True).raw))
-placeholder += f"<|image_1|>\n"
+path = Path("./data/raw/pdfs")
+i = 1
+for file in path.glob("*.pdf"):
+    pdf_images = convert_from_path(file)
+    for image in pdf_images:
+        images.append(image)
+        placeholder += f"<|image_{i}|>\n"
+        i += 1
 
 messages = [
-    {"role": "user", "content": placeholder + "Summarize the deck of slides."},
+    {
+        "role": "user",
+        "content": """
+<|image_1|>\nThis image shows an extract from a planning response form filled out by a member of the public. They may be pro or against the planning proposal. These planning applications typically cover the construction of new buildings, or similar infrastructure.
+
+Extract all structured information from these documents. For example a section may include a questionnaire that may or may not have been filled in. Please indicate the response from the member of public in a structured format, following the convention:
+
+{"<question>": "<response>"}
+
+The document may also include hand written notes under the title 'Your comments:', also include these notes verbatim, in a structured format. If a word is unreadable please use the special token <UNKNOWN>. Do not attempt to fill in the word if you are unsure.
+""",
+    },
 ]
 
 prompt = processor.tokenizer.apply_chat_template(
     messages, tokenize=False, add_generation_prompt=True
 )
 
-inputs = processor(prompt, images, return_tensors="pt").to("cuda:0")
+inputs = processor(prompt, images[0], return_tensors="pt").to("cuda:0")
 
-generation_args = {
-    "max_new_tokens": 1000,
-    "temperature": 0.0,
-    "do_sample": False,
-}
+generation_args = {"max_new_tokens": 1000, "do_sample": False}
 
 generate_ids = model.generate(
     **inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args

uv.lock

@@ -1066,6 +1066,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650 },
 ]
 
+[[package]]
+name = "pdf2image"
+version = "1.17.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pillow" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/00/d8/b280f01045555dc257b8153c00dee3bc75830f91a744cd5f84ef3a0a64b1/pdf2image-1.17.0.tar.gz", hash = "sha256:eaa959bc116b420dd7ec415fcae49b98100dda3dd18cd2fdfa86d09f112f6d57", size = 12811 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/62/33/61766ae033518957f877ab246f87ca30a85b778ebaad65b7f74fa7e52988/pdf2image-1.17.0-py3-none-any.whl", hash = "sha256:ecdd58d7afb810dffe21ef2b1bbc057ef434dabbac6c33778a38a3f7744a27e2", size = 11618 },
+]
+
 [[package]]
 name = "pexpect"
 version = "4.9.0"
@@ -1147,8 +1159,11 @@ dependencies = [
     { name = "langchain-community" },
     { name = "langchain-core" },
     { name = "langchain-unstructured" },
+    { name = "pdf2image" },
     { name = "pillow" },
+    { name = "setuptools" },
     { name = "torch" },
+    { name = "torchvision" },
     { name = "transformers" },
 ]
 
@@ -1165,8 +1180,11 @@ requires-dist = [
     { name = "langchain-community", specifier = ">=0.2.16" },
     { name = "langchain-core", specifier = ">=0.2.38" },
     { name = "langchain-unstructured", specifier = ">=0.1.2" },
+    { name = "pdf2image" },
     { name = "pillow", specifier = ">=10.4.0" },
+    { name = "setuptools" },
     { name = "torch", specifier = ">=2.4.1" },
+    { name = "torchvision", specifier = ">=0.19.1" },
     { name = "transformers", specifier = ">=4.44.2" },
 ]
 
@@ -1523,6 +1541,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e7/50/89e5eac4120b55422450d5221c86d526ace14e222ea3f6c0c005f8f011ec/safetensors-0.4.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:2c42e9b277513b81cf507e6121c7b432b3235f980cac04f39f435b7902857f91", size = 606993 },
 ]
 
+[[package]]
+name = "setuptools"
+version = "74.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3e/2c/f0a538a2f91ce633a78daaeb34cbfb93a54bd2132a6de1f6cec028eee6ef/setuptools-74.1.2.tar.gz", hash = "sha256:95b40ed940a1c67eb70fc099094bd6e99c6ee7c23aa2306f4d2697ba7916f9c6", size = 1356467 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/9c/9ad11ac06b97e55ada655f8a6bea9d1d3f06e120b178cd578d80e558191d/setuptools-74.1.2-py3-none-any.whl", hash = "sha256:5f4c08aa4d3ebcb57a50c33b1b07e94315d7fc7230f7115e47fc99776c8ce308", size = 1262071 },
+]
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -1715,6 +1742,30 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ac/30/8b6f77ea4ce84f015ee024b8dfef0dac289396254e8bfd493906d4cbb848/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d", size = 62123443 },
 ]
 
+[[package]]
+name = "torchvision"
+version = "0.19.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+    { name = "pillow" },
+    { name = "torch" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d4/90/cab820b96d4d1a36b088774209d2379cf49eda8210c8fee13552383860b7/torchvision-0.19.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:54e8513099e6f586356c70f809d34f391af71ad182fe071cc328a28af2c40608", size = 1660236 },
+    { url = "https://files.pythonhosted.org/packages/72/55/e0b3821c5595a9a2c8ec98d234b4a0d1142d91daac61f007503d3158f857/torchvision-0.19.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:20a1f5e02bfdad7714e55fa3fa698347c11d829fa65e11e5a84df07d93350eed", size = 7026373 },
+    { url = "https://files.pythonhosted.org/packages/db/71/da0f71c2765feee125b1dc280a6432aa88c510aedf9a36987f3fe7ed05ea/torchvision-0.19.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:7b063116164be52fc6deb4762de7f8c90bfa3a65f8d5caf17f8e2d5aadc75a04", size = 14072253 },
+    { url = "https://files.pythonhosted.org/packages/f7/8e/cbae11f8046d433881b478afc9e7589a76158124779cbc3a40163ec716bf/torchvision-0.19.1-cp310-cp310-win_amd64.whl", hash = "sha256:f40b6acabfa886da1bc3768f47679c61feee6bde90deb979d9f300df8c8a0145", size = 1288329 },
+    { url = "https://files.pythonhosted.org/packages/66/f6/a2f07a3f5385b37c45b8e14448b8610a8618dfad18ea437cb23b4edc50c5/torchvision-0.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:40514282b4896d62765b8e26d7091c32e17c35817d00ec4be2362ea3ba3d1787", size = 1660235 },
+    { url = "https://files.pythonhosted.org/packages/28/9d/40d1b943bbbd02a30d6b4f691d6de37a7e4c92f90bed0f8f47379e90eec6/torchvision-0.19.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:5a91be061ae5d6d5b95e833b93e57ca4d3c56c5a57444dd15da2e3e7fba96050", size = 7026152 },
+    { url = "https://files.pythonhosted.org/packages/36/04/36e1d35b864f4a7c8f3056a427542b14b3bcdbc66edd36faadee109b86c5/torchvision-0.19.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d71a6a6fe3a5281ca3487d4c56ad4aad20ff70f82f1d7c79bcb6e7b0c2af00c8", size = 14072255 },
+    { url = "https://files.pythonhosted.org/packages/f8/69/dc769cf54df8e828c0b8957b4521f35178f5bd4cc5b8fbe8a37ffd89a27c/torchvision-0.19.1-cp311-cp311-win_amd64.whl", hash = "sha256:70dea324174f5e9981b68e4b7cd524512c106ba64aedef560a86a0bbf2fbf62c", size = 1288330 },
+    { url = "https://files.pythonhosted.org/packages/a4/d0/b1029ab95d9219cac2dfc0d835e9ab4cebb01f5cb6b48e736778020fb995/torchvision-0.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27ece277ff0f6cdc7fed0627279c632dcb2e58187da771eca24b0fbcf3f8590d", size = 1660230 },
+    { url = "https://files.pythonhosted.org/packages/8b/34/fdd2d9e01228a069b28473a7c020bf1812c8ecab8565666feb247659ed30/torchvision-0.19.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:c659ff92a61f188a1a7baef2850f3c0b6c85685447453c03d0e645ba8f1dcc1c", size = 7026404 },
+    { url = "https://files.pythonhosted.org/packages/da/b2/9da42d67dfc30d9e3b161f7a37f6c7eca86a80e6caef4a9aa11727faa4f5/torchvision-0.19.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:c07bf43c2a145d792ecd9d0503d6c73577147ece508d45600d8aac77e4cdfcf9", size = 14072022 },
+    { url = "https://files.pythonhosted.org/packages/6b/b2/fd577e1622b43cdeb74782a60cea4909f88f471813c215ea7b4e7ea84a74/torchvision-0.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b4283d283675556bb0eae31d29996f53861b17cbdcdf3509e6bc050414ac9289", size = 1288328 },
+]
+
 [[package]]
 name = "tqdm"
 version = "4.66.5"