\u914d\u7f6e OpenAI API \u5bc6\u94a5\u3002<\/li>\n<\/ul>\n\u793a\u4f8b\u4ee3\u7801\uff1a<\/p>\n
from langchain_core.prompts import ChatPromptTemplate\r\nfrom langchain_openai import ChatOpenAI\r\nfrom langchain_core.pydantic_v1 import BaseModel, Field\r\nfrom langchain import hub\r\n# 1. \u6587\u672c\u547d\u9898\u5316 (Propositioning)\r\n# \u793a\u4f8b\u6587\u672c\r\ntext = \"\"\"\r\nOn July 20, 1969, astronaut Neil Armstrong walked on the moon.\r\nHe was leading NASA's Apollo 11 mission.\r\nArmstrong famously said, \"That's one small step for man, one giant leap for mankind\" as he stepped onto the lunar surface.\r\nLater, he planted the American flag.\r\nThe mission was a success.\r\n\"\"\"\r\n# \u4ece Langchain hub \u83b7\u53d6\u547d\u9898\u5316\u63d0\u793a\u6a21\u677f\r\nobj = hub.pull(\"wfh\/proposal-indexing\")\r\n# \u4f7f\u7528 GPT-4o \u6a21\u578b\r\nllm = ChatOpenAI(model=\"gpt-4o\")\r\n# \u5b9a\u4e49 Pydantic \u6a21\u578b\u4ee5\u63d0\u53d6\u53e5\u5b50\r\nclass Sentences(BaseModel):\r\nsentences: list[str]\r\n# \u521b\u5efa\u7ed3\u6784\u5316\u8f93\u51fa\u7684 LLM\r\nextraction_llm = llm.with_structured_output(Sentences)\r\n# \u521b\u5efa\u53e5\u5b50\u63d0\u53d6\u94fe\r\nextraction_chain = obj | extraction_llm\r\n# \u5c06\u6587\u672c\u5206\u5272\u6210\u6bb5\u843d (\u4e3a\u7b80\u5316\u793a\u4f8b\uff0c\u672c\u6587\u5047\u8bbe\u8f93\u5165\u6587\u672c\u4ec5\u5305\u542b\u4e00\u4e2a\u6bb5\u843d\uff0c\u5b9e\u9645\u5e94\u7528\u4e2d\u53ef\u5904\u7406\u591a\u6bb5\u843d\u6587\u672c\u3002)\r\nparagraphs = [text]\r\npropositions = []\r\nfor p in paragraphs:\r\nsentences = extraction_chain.invoke(p)\r\npropositions.extend(sentences.sentences)\r\nprint(\"Propositions:\", propositions)\r\n# 2. \u521b\u5efa LLM Agent\r\n# \u5b9a\u4e49\u5757\u5143\u6570\u636e\u6a21\u578b\r\nclass ChunkMeta(BaseModel):\r\ntitle: str = Field(description=\"The title of the chunk.\")\r\nsummary: str = Field(description=\"The summary of the chunk.\")\r\n# \u7528\u4e8e\u751f\u6210\u6458\u8981\u548c\u6807\u9898\u7684 LLM (\u8fd9\u91cc\u53ef\u4ee5\u4f7f\u7528\u6e29\u5ea6\u8f83\u4f4e\u7684\u6a21\u578b)\r\nsummary_llm = ChatOpenAI(temperature=0)\r\n# \u7528\u4e8e\u5757\u5206\u914d\u7684 LLM\r\nallocation_llm = ChatOpenAI(temperature=0)\r\n# \u5b58\u50a8\u5df2\u521b\u5efa\u7684\u6587\u672c\u5757\u7684\u5b57\u5178\r\nchunks = {}\r\n# 3. \u521b\u5efa\u65b0\u5757\u7684\u51fd\u6570\r\ndef create_new_chunk(chunk_id, proposition):\r\nsummary_prompt_template = ChatPromptTemplate.from_messages(\r\n[\r\n(\r\n\"system\",\r\n\"Generate a new summary and a title based on the propositions.\",\r\n),\r\n(\r\n\"user\",\r\n\"propositions:{propositions}\",\r\n),\r\n]\r\n)\r\nsummary_chain = summary_prompt_template | summary_llm\r\nchunk_meta = summary_chain.invoke(\r\n{\r\n\"propositions\": [proposition],\r\n}\r\n)\r\nchunks[chunk_id] = {\r\n\"chunk_id\": chunk_id, # \u6dfb\u52a0 chunk_id\r\n\"summary\": chunk_meta.summary,\r\n\"title\": chunk_meta.title,\r\n\"propositions\": [proposition],\r\n}\r\nreturn chunk_id # \u8fd4\u56de chunk_id\r\n# 4. \u5c06\u547d\u9898\u6dfb\u52a0\u5230\u73b0\u6709\u5757\u7684\u51fd\u6570\r\ndef add_proposition(chunk_id, proposition):\r\nsummary_prompt_template = ChatPromptTemplate.from_messages(\r\n[\r\n(\r\n\"system\",\r\n\"If the current_summary and title is still valid for the propositions, return them.\"\r\n\"If not, generate a new summary and a title based on the propositions.\",\r\n),\r\n(\r\n\"user\",\r\n\"current_summary:{current_summary}\\ncurrent_title:{current_title}\\npropositions:{propositions}\",\r\n),\r\n]\r\n)\r\nsummary_chain = summary_prompt_template | summary_llm\r\nchunk = chunks[chunk_id]\r\ncurrent_summary = chunk[\"summary\"]\r\ncurrent_title = chunk[\"title\"]\r\ncurrent_propositions = chunk[\"propositions\"]\r\nall_propositions = current_propositions + [proposition]\r\nchunk_meta = summary_chain.invoke(\r\n{\r\n\"current_summary\": current_summary,\r\n\"current_title\": current_title,\r\n\"propositions\": all_propositions,\r\n}\r\n)\r\nchunk[\"summary\"] = chunk_meta.summary\r\nchunk[\"title\"] = chunk_meta.title\r\nchunk[\"propositions\"] = all_propositions\r\n# 5. Agent \u7684\u6838\u5fc3\u903b\u8f91\r\ndef find_chunk_and_push_proposition(proposition):\r\nclass ChunkID(BaseModel):\r\nchunk_id: int = Field(description=\"The chunk id.\")\r\nallocation_prompt = ChatPromptTemplate.from_messages(\r\n[\r\n(\r\n\"system\",\r\n\"You have the chunk ids and the summaries. \"\r\n\"Find the chunk that best matches the proposition. \"\r\n\"If no chunk matches, return a new chunk id. \"\r\n\"Return only the chunk id.\",\r\n),\r\n(\r\n\"user\",\r\n\"proposition:{proposition}\\nchunks_summaries:{chunks_summaries}\",\r\n),\r\n]\r\n)\r\nallocation_chain = allocation_prompt | allocation_llm.with_structured_output(ChunkID)\r\nchunks_summaries = {\r\nchunk_id: chunk[\"summary\"] for chunk_id, chunk in chunks.items()\r\n}\r\n# \u521d\u59cbchunks\u53ef\u80fd\u4e3a\u7a7a\uff0c\u5bfc\u81f4allocation_chain.invoke\u62a5\u9519\r\nif not chunks_summaries:\r\n# \u5982\u679c\u6ca1\u6709\u5df2\u5b58\u5728\u7684\u5757\uff0c\u76f4\u63a5\u521b\u5efa\u65b0\u5757\r\nnext_chunk_id = 1\r\ncreate_new_chunk(next_chunk_id, proposition)\r\nreturn\r\nbest_chunk_id = allocation_chain.invoke(\r\n{\"proposition\": proposition, \"chunks_summaries\": chunks_summaries}\r\n).chunk_id\r\nif best_chunk_id not in chunks:\r\n# \u5982\u679c\u8fd4\u56de\u7684 chunk_id \u4e0d\u5b58\u5728\uff0c\u521b\u5efa\u65b0\u5757\r\nnext_chunk_id = max(chunks.keys(), default=0) + 1 if chunks else 1\r\ncreate_new_chunk(next_chunk_id, proposition)\r\nelse:\r\nadd_proposition(best_chunk_id, proposition)\r\n# \u904d\u5386\u547d\u9898\u5217\u8868\uff0c\u8fdb\u884c\u5206\u5757\r\nfor i, proposition in enumerate(propositions):\r\nfind_chunk_and_push_proposition(proposition)\r\n# \u6253\u5370\u6700\u7ec8\u7684\u5757\r\nprint(\"\\nFinal Chunks:\")\r\nfor chunk_id, chunk in chunks.items():\r\nprint(f\"Chunk {chunk_id}:\")\r\nprint(f\" Title: {chunk['title']}\")\r\nprint(f\" Summary: {chunk['summary']}\")\r\nprint(f\" Propositions: {chunk['propositions']}\")\r\nprint(\"-\" * 20)\r\n<\/code><\/pre>\n\u4ee3\u7801\u89e3\u91ca\uff1a<\/p>\n
\n- \u547d\u9898\u5316\uff1a\n
\n- \u4ee3\u7801\u793a\u4f8b\u9996\u5148\u4f7f\u7528 hub.pull(“wfh\/proposal-indexing”) \u4ece Langchain Hub \u52a0\u8f7d\u9884\u8bbe\u7684\u547d\u9898\u5316\u63d0\u793a\u6a21\u677f\u3002<\/li>\n
- \u63a5\u7740\uff0c\u4f7f\u7528 ChatOpenAI(model=”gpt-4o”) \u521d\u59cb\u5316 LLM \u5b9e\u4f8b\uff0c\u9009\u7528 GPT-4o \u6a21\u578b\u4ee5\u83b7\u5f97\u66f4\u4f18\u7684\u6027\u80fd\u3002<\/li>\n
- \u5b9a\u4e49 Sentences Pydantic \u6a21\u578b\uff0c\u7528\u4e8e\u7ed3\u6784\u5316\u89e3\u6790 LLM \u8f93\u51fa\u7684\u53e5\u5b50\u5217\u8868\u3002<\/li>\n
- \u6784\u5efa\u63d0\u793a\u6a21\u677f\u4e0e LLM \u7684\u8fde\u63a5\u94fe extraction_chain\u3002<\/li>\n
- \u4e3a\u7b80\u5316\u793a\u4f8b\uff0c\u672c\u6587\u5047\u8bbe\u8f93\u5165\u6587\u672c\u4ec5\u5305\u542b\u4e00\u4e2a\u6bb5\u843d\uff0c\u5b9e\u9645\u5e94\u7528\u4e2d\u53ef\u5904\u7406\u591a\u6bb5\u843d\u6587\u672c\u3002 \u4ee3\u7801\u5c06\u793a\u4f8b\u6587\u672c\u5206\u5272\u6210\u6bb5\u843d\u5217\u8868\u3002<\/li>\n
- \u5faa\u73af\u904d\u5386\u6bb5\u843d\uff0c\u5e76\u4f7f\u7528 extraction_chain \u5c06\u6bb5\u843d\u8f6c\u5316\u4e3a\u547d\u9898\u5217\u8868\u3002<\/li>\n<\/ul>\n<\/li>\n
- \u521b\u5efa LLM Agent\uff1a\n
\n- \u5b9a\u4e49 ChunkMeta Pydantic \u6a21\u578b\uff0c\u5b9a\u4e49\u5757\u5143\u6570\u636e\u7ed3\u6784\uff08\u6807\u9898\u548c\u6458\u8981\uff09\u3002<\/li>\n
- \u521b\u5efa summary_llm \u548c allocation_llm \u4e24\u4e2a LLM \u5b9e\u4f8b\u3002 summary_llm \u7528\u4e8e\u751f\u6210\u6587\u672c\u5757\u7684\u6458\u8981\u548c\u6807\u9898\uff0c\u800c allocation_llm \u5219\u8d1f\u8d23\u5224\u65ad\u547d\u9898\u5e94\u5f52\u5165\u54ea\u4e2a\u5df2\u6709\u7684\u5757\u6216\u521b\u5efa\u65b0\u5757\u3002<\/li>\n
- \u521d\u59cb\u5316 chunks \u5b57\u5178\uff0c\u7528\u4e8e\u5b58\u50a8\u5df2\u521b\u5efa\u7684\u6587\u672c\u5757\u3002<\/li>\n<\/ul>\n<\/li>\n
- create_new_chunk \u51fd\u6570\uff1a\n
\n- \u51fd\u6570\u63a5\u53d7 chunk_id \u548c proposition \u4f5c\u4e3a\u8f93\u5165\u53c2\u6570\u3002<\/li>\n
- \u57fa\u4e8e\u547d\u9898\uff0c\u5229\u7528 summary_prompt_template \u548c summary_llm \u751f\u6210\u5757\u7684\u6807\u9898\u548c\u6458\u8981\u3002<\/li>\n
- \u5e76\u5c06\u65b0\u5757\u5b58\u5165 chunks \u5b57\u5178\u3002<\/li>\n<\/ul>\n<\/li>\n
- add_proposition \u51fd\u6570\uff1a\n
\n- \u51fd\u6570\u540c\u6837\u63a5\u6536 chunk_id \u548c proposition \u4f5c\u4e3a\u8f93\u5165\u3002<\/li>\n
- \u4ece chunks \u5b57\u5178\u68c0\u7d22\u73b0\u6709\u5757\u4fe1\u606f\u3002<\/li>\n
- \u66f4\u65b0\u5f53\u524d\u5757\u7684\u547d\u9898\u5217\u8868\u3002<\/li>\n
- \u91cd\u65b0\u8bc4\u4f30\u5e76\u66f4\u65b0\u5757\u7684\u6807\u9898\u548c\u6458\u8981\u3002<\/li>\n
- \u5e76\u66f4\u65b0 chunks \u5b57\u5178\u4e2d\u5bf9\u5e94\u5757\u7684\u5143\u6570\u636e\u3002<\/li>\n<\/ul>\n<\/li>\n
- find_chunk_and_push_proposition \u51fd\u6570 (Agent \u6838\u5fc3\u903b\u8f91)\uff1a\n
\n- \u5b9a\u4e49 ChunkID Pydantic \u6a21\u578b\uff0c\u7528\u4e8e\u89e3\u6790 LLM \u8f93\u51fa\u7684\u5757 ID\u3002<\/li>\n
- \u521b\u5efa allocation_prompt\uff0c\u6307\u793a LLM \u5bfb\u627e\u4e0e\u5f53\u524d\u547d\u9898\u6700\u5339\u914d\u7684\u73b0\u6709\u5757\uff0c\u6216\u8fd4\u56de\u65b0\u7684\u5757 ID\u3002<\/li>\n
- \u6784\u5efa allocation_chain\uff0c\u8fde\u63a5\u63d0\u793a\u6a21\u677f\u548c allocation_llm\u3002<\/li>\n
- \u6784\u5efa chunks_summaries \u5b57\u5178\uff0c\u5176\u4e2d\u5b58\u50a8\u4e86\u73b0\u6709\u5757\u7684 ID \u548c\u6458\u8981\u4fe1\u606f\u3002<\/li>\n
- \u82e5 chunks \u5b57\u5178\u4e3a\u7a7a\uff08\u5373\u5c1a\u65e0\u4efb\u4f55\u6587\u672c\u5757\uff09\uff0c\u5219\u76f4\u63a5\u521b\u5efa\u65b0\u5757\u3002<\/li>\n
- \u4f7f\u7528 allocation_chain \u8c03\u7528 LLM\uff0c\u4ee5\u83b7\u53d6\u6700\u4f73\u5339\u914d\u5757\u7684 ID\u3002<\/li>\n
- \u5982\u679c LLM \u8fd4\u56de\u7684 chunk_id \u4e0d\u5728 chunks \u5b57\u5178\u4e2d\uff0c\u8868\u660e\u9700\u8981\u521b\u5efa\u65b0\u7684\u6587\u672c\u5757\uff0c\u6b64\u65f6\u8c03\u7528 create_new_chunk \u51fd\u6570\u3002<\/li>\n
- \u5982\u679c\u8fd4\u56de\u7684 chunk_id \u5df2\u5b58\u5728\u4e8e chunks \u5b57\u5178\u4e2d\uff0c\u8868\u660e\u5e94\u5c06\u5f53\u524d\u547d\u9898\u6dfb\u52a0\u5230\u73b0\u6709\u6587\u672c\u5757\u4e2d\uff0c\u8c03\u7528 add_proposition \u51fd\u6570\u3002<\/li>\n<\/ul>\n<\/li>\n
- \u4e3b\u5faa\u73af\uff1a\n
\n- \u5faa\u73af\u904d\u5386\u547d\u9898\u5217\u8868\u3002<\/li>\n
- \u9488\u5bf9\u6bcf\u4e2a\u547d\u9898\uff0c\u8c03\u7528 find_chunk_and_push_proposition \u51fd\u6570\uff0c\u5e76\u5c06\u547d\u9898\u5206\u914d\u81f3\u76f8\u5e94\u7684\u6587\u672c\u5757\u3002<\/li>\n<\/ul>\n<\/li>\n
- \u8f93\u51fa\u7ed3\u679c\uff1a\n