How to Optimize LLM Applications With Prompt Compression Using LLMLingua and LangChain

1
from llmlingua import PromptCompressor
2
3
llm_lingua = PromptCompressor(
4
    model_name="microsoft/llmlingua-2-bert-base-multilingual-cased-meetingbank",
5
    model_config={"revision": "main"},
6
    use_llmlingua2=True,
7
    device_map="cpu"
8
)

1
# Function definition
2
def compress_query_prompt(context):
3
4
  compressed_prompt = llm_lingua.compress_prompt(
5
    str(context),
6
    rate=0.33,
7
    force_tokens=["!", ".", "?", "\n"],
8
    drop_consecutive=True,
9
  )
10
11
  return compressed_prompt

1
import pprint
2
3
def handle_user_query_with_compression(query, collection):
4
5
  get_knowledge = vector_search(query, collection)
6
  
7
  # Concatenate the search results to reflect the employee profile
8
  search_result = ''
9
10
  for result in get_knowledge:      
11
      employee_profile = f"""
12
      Employee ID: {result.get('employee_id', 'N/A')}
13
      Name: {result.get('first_name', 'N/A')} {result.get('last_name', 'N/A')}
14
      Gender: {result.get('gender', 'N/A')}
15
      Date of Birth: {result.get('date_of_birth', 'N/A')}
16
      """
17
      search_result += employee_profile + "\n"
18
19
  # Prepare information for compression
20
  query_info = {
21
    'demonstration_str': search_result,  # Results from information retrieval process
22
    'instruction': "Write a high-quality answer for the given question using only the provided search results.",
23
    'question': query
24
  }
25
26
  # Compress the query prompt
27
  compressed_prompt = compress_query_prompt(query_info)
28
  
29
  prompt =  f"Answer this user query: {query} with the following context:\n{compressed_prompt}"
30
  print("Compressed Prompt:\n")
31
  pprint.pprint(prompt)
32
33
  completion = openai.chat.completions.create(
34
      model=OPEN_AI_MODEL,
35
      messages=[
36
          {"role": "system", "content": "You are a Human Resource System within a corporate company."},
37
          {"role": "user", "content": prompt}
38
      ]
39
  )
40
41
  return (completion.choices[0].message.content), search_result

1
from langchain.retrievers import ContextualCompressionRetriever
2
from langchain_community.document_compressors import LLMLinguaCompressor
3
4
compressor = LLMLinguaCompressor(model_name="openai-community/gpt2", device_map="cpu")
5
compression_retriever = ContextualCompressionRetriever(
6
    base_compressor=compressor, base_retriever=retriever
7
)
8
9
compressed_docs = compression_retriever.invoke(
10
    "Who is the CEO?"
11
)
12
print(compressed_docs)

1
from langchain.chains import RetrievalQA
2
3
chain = RetrievalQA.from_chain_type(llm=llm, retriever=compression_retriever)
4
chain.invoke({"query": "Who is the CEO?"})

1
from langchain.retrievers import ContextualCompressionRetriever
2
from langchain_community.document_compressors import LLMLinguaCompressor
3
from langchain.agents import AgentExecutor, create_tool_calling_agent
4
5
compressor = LLMLinguaCompressor(model_name="openai-community/gpt2", device_map="cpu")
6
7
@tool
8
def compress_prompt_using_llmlingua(prompt: str, compression_rate: float = 0.5) -> str:
9
    """
10
    Compresses a long data or prompt using the LLMLinguaCompressor.
11
12
    Args:
13
        data (str): The data or prompt to be compressed.
14
        compression_rate (float): The rate at which to compress the data (default is 0.5).
15
16
    Returns:
17
        str: The compressed data or prompt.
18
    """
19
    compressed_data = compressor.compress_prompt(
20
        prompt,
21
        rate=compression_rate,
22
        force_tokens=["!", ".", "?", "\n"],
23
        drop_consecutive=True
24
    )
25
    return compressed_data
26
27
28
29
tools = [compress_prompt_using_llmlingua]
30
31
agent = create_tool_calling_agent(llm, tools, prompt)
32
33
agent_executor = AgentExecutor(
34
    agent=agent,
35
    tools=tools,
36
    verbose=True,
37
    handle_parsing_errors=True,
38
    memory=memory,
39
)
40
41
agent_executor.invoke({"input": "Get me a list of research papers on the topic Prompt Compression"})

1
from langchain.retrievers import ContextualCompressionRetriever
2
from langchain_community.document_compressors import LLMLinguaCompressor
3
from langchain.tools.retriever import create_retriever_tool
4
5
compressor = LLMLinguaCompressor(model_name="openai-community/gpt2", device_map="cpu")
6
compression_retriever = ContextualCompressionRetriever(
7
    base_compressor=compressor, base_retriever=retriever
8
)
9
10
retriever_tool = create_retriever_tool(
11
    retriever=retriever,
12
    name="knowledge_base",
13
    description="This serves as the base knowledge source of the agent and contains some records of research papers from Arxiv. This tool is used as the first step for exploration and research efforts."
14
)