ADD: LLM techs

README.md

@@ -45,4 +45,23 @@ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
 
 import torch
 torch.cuda.empty_cache()
-torch.cuda.reset_peak_memory_stats()
+torch.cuda.reset_peak_memory_stats()
+
+7/12
+# 1 Fine‑tune image model LoRA+QLoRA
+python train_lora.py
+
+# 2 SFT 语言模型
+python sft_train.py
+
+# 3 Build RAG index
+python build_embeddings.py
+
+# 4 (可选) 收集偏好 → 训练 reward model
+python reward_model.py
+
+# 5 PPO RLHF 微调
+python ppo_tune.py
+
+# 6 Inference with RAG
+python rag_infer.py

build_embeddings.py

@@ -0,0 +1,11 @@
+from sentence_transformers import SentenceTransformer
+import faiss, json, glob, os, numpy as np
+
+model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
+texts=[]; vecs=[]
+for f in glob.glob("nyc_ads_dataset/*.json"):
+    cap=json.load(open(f))["caption"]
+    texts.append(cap); vecs.append(model.encode(cap,normalize_embeddings=True))
+vecs=np.vstack(vecs).astype("float32")
+index=faiss.IndexFlatIP(vecs.shape[1]); index.add(vecs)
+faiss.write_index(index,"prompt.index"); json.dump(texts,open("prompt.txt","w"))

inference.py

@@ -1,3 +1,4 @@
+'''
 from diffusers import StableDiffusionPipeline
 import torch
 
@@ -12,4 +13,85 @@ image = pipe(prompt, num_inference_steps=500, guidance_scale=7.5).images[0]
 
 # Display or save the image
 image.save("output_nyc_ad.png")
-image.show()
+image.show()
+'''
+'''
+import torch, faiss, json
+from sentence_transformers import SentenceTransformer
+from diffusers import StableDiffusionPipeline
+
+texts=json.load(open("prompt.txt"))
+index=faiss.read_index("prompt.index")
+emb=SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
+pipe=StableDiffusionPipeline.from_pretrained("./nyc-ad-model",torch_dtype=torch.float16).to("cuda")
+
+def rag_prompt(query,k=3):
+    q=emb.encode(query,normalize_embeddings=True).astype("float32")
+    _,I=index.search(q.reshape(1,-1),k)
+    retrieved=" ".join(texts[i] for i in I[0])
+    return f"{retrieved}. {query}"
+
+prompt=rag_prompt("fried chicken advertisement poster")
+img=pipe(prompt,num_inference_steps=30,guidance_scale=7.5).images[0]
+img.save("rag_output.png")
+'''
+
+import torch, faiss, json
+from sentence_transformers import SentenceTransformer
+from diffusers import StableDiffusionPipeline
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+# Load RAG index
+texts = json.load(open("prompt.txt"))
+index = faiss.read_index("prompt.index")
+emb = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
+
+# Load image generation pipeline
+pipe = StableDiffusionPipeline.from_pretrained(
+    "./nyc-ad-model",
+    torch_dtype=torch.float16
+).to("cuda")
+
+# Load your own fine-tuned SFT model
+text_model_path = "./sft-model"  # Path to your SFT-finetuned model
+tokenizer = AutoTokenizer.from_pretrained(text_model_path)
+text_model = AutoModelForCausalLM.from_pretrained(
+    text_model_path,
+    torch_dtype=torch.float16,
+    device_map="auto"
+)
+
+# Build retrieval-augmented prompt
+def rag_prompt(query, k=3):
+    q = emb.encode(query, normalize_embeddings=True).astype("float32")
+    _, I = index.search(q.reshape(1, -1), k)
+    retrieved = " ".join(texts[i] for i in I[0])
+    return f"{retrieved}. {query}"
+
+# Prompt for generation
+user_prompt = "fried chicken advertisement poster"
+full_prompt = rag_prompt(user_prompt)
+
+# Generate image
+image = pipe(full_prompt, num_inference_steps=30, guidance_scale=7.5).images[0]
+image.save("rag_output.png")
+
+# Construct input prompt compatible with SFT format
+copy_prompt = f"""### Instruction:
+Generate a catchy advertisement slogan for: {user_prompt}
+
+### Response:"""
+
+inputs = tokenizer(copy_prompt, return_tensors="pt").to("cuda")
+output_ids = text_model.generate(
+    **inputs,
+    max_new_tokens=30,
+    do_sample=True,
+    top_p=0.95
+)
+response = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+
+# Output result
+print("🖼️ Image saved to rag_output.png")
+print("📝 Generated slogan:")
+print(response.strip())

ppo_tune.py

@@ -0,0 +1,19 @@
+from trl import PPOTrainer, PPOConfig
+from peft import PeftModel
+import torch, random, json, glob
+from diffusers import StableDiffusionPipeline
+from reward_model import CLIPModel, CLIPProcessor
+
+rm=CLIPModel.from_pretrained("rm").eval().half().cuda()
+proc=CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+pipe=StableDiffusionPipeline.from_pretrained("./nyc-ad-model",torch_dtype=torch.float16).to("cuda")
+ppo_cfg=PPOConfig(batch_size=1,learning_rate=1e-6,target_kl=0.2)
+trainer=PPOTrainer(model=pipe.unet, reward_model=rm, config=ppo_cfg)
+
+prompts=[l.strip() for l in open("prompt.txt")]
+for step in range(500):
+    p=random.choice(prompts)
+    img=pipe(p,num_inference_steps=20).images[0]
+    reward=rm(**proc(text=p,images=img,return_tensors="pt").to("cuda")).logits[0,0].item()
+    trainer.step(prompts=[p], rewards=[reward])
+pipe.save_pretrained("nyc-ad-model-rlhf")

requirements.txt

@@ -6,4 +6,11 @@ transformers
 xformers
 torchvision
 flickrapi
-requests
+requests
+peft>=0.9.0
+bitsandbytes
+faiss-cpu
+sentence-transformers
+trl[peft]
+label-studio
+datasets

reward_model.py

@@ -0,0 +1,21 @@
+from transformers import CLIPProcessor, CLIPModel, TrainingArguments, Trainer
+import datasets, torch, json, glob
+
+model=CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+processor=CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+data=[]
+for f in glob.glob("human_prefs/*.json"):
+    j=json.load(open(f)); data.append(j)  # {"prompt":…, "good":img_path, "bad":img_path}
+
+dataset=datasets.Dataset.from_list(data)
+
+def preprocess(ex):
+    inputs=processor(text=[ex["prompt"]*2], images=[ex["good"],ex["bad"]], return_tensors="pt")
+    inputs["labels"]=torch.tensor([1,0])
+    return inputs
+
+dataset=dataset.map(preprocess,remove_columns=dataset.column_names)
+args=TrainingArguments("rm_ckpt",per_device_train_batch_size=2,fp16=True,learning_rate=5e-6,epochs=3)
+trainer=Trainer(model,args,train_dataset=dataset)
+trainer.train(); model.save_pretrained("rm")

sft_train.py

@@ -0,0 +1,41 @@
+import torch, json
+from datasets import load_dataset, Dataset
+from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForLanguageModeling
+from peft import get_peft_model, LoraConfig, TaskType
+
+# Load your dataset
+data = [json.loads(l) for l in open("data/sft_data.jsonl")]
+dataset = Dataset.from_list(data)
+
+# Load model & tokenizer
+base_model = "meta-llama/Llama-2-7b-hf"  # Or use Mistral, Falcon, etc.
+tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast=True)
+model = AutoModelForCausalLM.from_pretrained(base_model, torch_dtype=torch.float16)
+
+# Add LoRA (optional)
+lora_config = LoraConfig(task_type=TaskType.CAUSAL_LM, r=8, lora_alpha=32, lora_dropout=0.05,
+                         target_modules=["q_proj", "v_proj"])
+model = get_peft_model(model, lora_config)
+
+# Preprocessing
+def tokenize(example):
+    prompt = f"### Instruction:\n{example['prompt']}\n\n### Response:\n{example['output']}"
+    return tokenizer(prompt, truncation=True, max_length=512, padding="max_length")
+dataset = dataset.map(tokenize, remove_columns=dataset.column_names)
+
+# Training setup
+args = TrainingArguments(
+    output_dir="./sft-model",
+    per_device_train_batch_size=2,
+    num_train_epochs=3,
+    fp16=True,
+    evaluation_strategy="no",
+    save_strategy="epoch",
+    logging_steps=20,
+    learning_rate=2e-5,
+    report_to="tensorboard",
+)
+
+data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+trainer = Trainer(model=model, args=args, train_dataset=dataset, data_collator=data_collator)
+trainer.train()