Chen commited on
Commit
09c542e
·
1 Parent(s): a03e89c

remove unused file

Browse files
Files changed (1) hide show
  1. github_retriever.py +0 -63
github_retriever.py DELETED
@@ -1,63 +0,0 @@
1
- from llama_hub.github_repo import GithubRepositoryReader, GithubClient
2
- from llama_index import download_loader, GPTVectorStoreIndex
3
- from llama_index import LLMPredictor, VectorStoreIndex, ServiceContext
4
- from langchain.llms import AzureOpenAI
5
- from langchain.embeddings.openai import OpenAIEmbeddings
6
- from llama_index import LangchainEmbedding, ServiceContext
7
- from llama_index import StorageContext, load_index_from_storage
8
- from dotenv import load_dotenv
9
- import os
10
- import pickle
11
-
12
-
13
- def main() -> None:
14
- # define embedding
15
- embedding = LangchainEmbedding(OpenAIEmbeddings(chunk_size=1))
16
- # define LLM
17
- llm_predictor = LLMPredictor(
18
- llm=AzureOpenAI(
19
- engine="text-davinci-003",
20
- model_name="text-davinci-003",
21
- )
22
- )
23
-
24
- # configure service context
25
- service_context = ServiceContext.from_defaults(
26
- llm_predictor=llm_predictor, embed_model=embedding
27
- )
28
- download_loader("GithubRepositoryReader")
29
- docs = None
30
- if os.path.exists("docs/docs.pkl"):
31
- with open("docs/docs.pkl", "rb") as f:
32
- docs = pickle.load(f)
33
-
34
- if docs is None:
35
- github_client = GithubClient(os.getenv("GITHUB_TOKEN"))
36
- loader = GithubRepositoryReader(
37
- github_client,
38
- owner="ctripcorp",
39
- repo="x-pipe",
40
- filter_directories=(
41
- [".", "doc"],
42
- GithubRepositoryReader.FilterType.INCLUDE,
43
- ),
44
- filter_file_extensions=([".md"], GithubRepositoryReader.FilterType.INCLUDE),
45
- verbose=True,
46
- concurrent_requests=10,
47
- )
48
-
49
- docs = loader.load_data(branch="master")
50
-
51
- with open("docs/docs.pkl", "wb") as f:
52
- pickle.dump(docs, f)
53
-
54
- index = GPTVectorStoreIndex.from_documents(docs, service_context=service_context)
55
-
56
- query_engine = index.as_query_engine(service_context=service_context)
57
- response = query_engine.query("如何使用X-Pipe?")
58
- print(response)
59
-
60
-
61
- if __name__ == "__main__":
62
- load_dotenv()
63
- main()