Planejador de partes favoritas do operador Joe com Playwright, LlamaIndex e MongoDB Atlas Vector Search

1
!pip install playwright
2
!playwright install

1
import asyncio
2
from playwright.async_api import async_playwright

1
async def traderJoesScraper():
2
   async with async_playwright() as playwright:
3
       # use headless mode since we are using Colab
4
       browser = await playwright.chromium.launch(headless=True)
5
       page = await browser.new_page()
6
7
8
       # all the URLs for my foods, bevs, flowers&plants, and everything else categories
9
       pages = [
10
           {'url': 'https://www.traderjoes.com/home/products/category/food-8?filters=%7B%22tags%22%3A%5B%22Fall+Faves%22%5D%7D', 'category': 'Food'},
11
           {'url': 'https://www.traderjoes.com/home/products/category/food-8?filters=%7B%22tags%22%3A%5B%22Fall+Faves%22%5D%2C%22page%22%3A2%7D', 'category': 'Food'},
12
           {'url': 'https://www.traderjoes.com/home/products/category/food-8?filters=%7B%22tags%22%3A%5B%22Fall+Faves%22%5D%2C%22page%22%3A3%7D', 'category': 'Food'},
13
           {'url': 'https://www.traderjoes.com/home/products/category/food-8?filters=%7B%22tags%22%3A%5B%22Fall+Faves%22%5D%2C%22page%22%3A4%7D', 'category': 'Food'},
14
           {'url': 'https://www.traderjoes.com/home/products/category/food-8?filters=%7B%22tags%22%3A%5B%22Fall+Faves%22%5D%2C%22page%22%3A5%7D', 'category': 'Food'},
15
           {'url': 'https://www.traderjoes.com/home/products/category/beverages-182?filters=%7B%22tags%22%3A%5B%22Fall+Faves%22%5D%7D', 'category': 'Beverage'},
16
           {'url': 'https://www.traderjoes.com/home/products/category/flowers-plants-203?filters=%7B%22tags%22%3A%5B%22Fall+Faves%22%5D%7D', 'category': 'Flowers&Plants'},
17
           {'url': 'https://www.traderjoes.com/home/products/category/everything-else-215?filters=%7B%22tags%22%3A%5B%22Fall+Faves%22%5D%7D', 'category': 'EverythingElse'}
18
       ]
19
20
21
       items = []
22
23
24
       # loop through each URL
25
       for info in pages:
26
           await page.goto(info['url'])
27
28
29
           # let page load
30
           await page.wait_for_selector('li.ProductList_productList__item__1EIvq', state='attached', timeout=60000)
31
32
33
           # li.ProductList_productList__item__1EIvq is where all our info lives
34
           products = await page.query_selector_all('li.ProductList_productList__item__1EIvq')
35
36
37
           # get all our info
38
           for product in products:
39
               result = {}
40
41
42
               name = await product.query_selector('h2.ProductCard_card__title__text__uiWLe a')
43
               price = await product.query_selector('span.ProductPrice_productPrice__price__3-50j')
44
45
46
               if name and price:
47
                   result['name'] = await name.inner_text()
48
49
50
                   # have to make price a number
51
                   price_text = await price.inner_text()
52
                   convert_price = float(price_text.replace('$', '').strip())
53
                   result['price'] = convert_price
54
55
56
                   # category is so we can save it nicely later
57
                   result['category'] = info['category']
58
                   items.append(result)
59
60
61
       for item in items:
62
           print(f"Name: {item['name']}, Price: {item['price']}, Category: {item['category']}")
63
64
65
       await browser.close()
66
       return items
67
68
69
70
71
scraped_products = await traderJoesScraper()
72
print(scraped_products)

1
scraped_products_count = len(scraped_products)
2
print(scraped_products_count)

1
with open('tj_fall_faves_oct30.txt', 'w') as f:
2
   for item in scraped_products:
3
       f.write(f"Name: {item['name']}, Price: ${item['price']}, Category: {item['category']}\n")

1
pip install --quiet --upgrade llama-index llama-index-vector-stores-mongodb llama-index-embeddings-openai pymongo

1
import getpass, os, pymongo, pprint
2
from pymongo.operations import SearchIndexModel
3
from llama_index.core import SimpleDirectoryReader, VectorStoreIndex, StorageContext
4
from llama_index.core.settings import Settings
5
from llama_index.core.retrievers import VectorIndexRetriever
6
from llama_index.core.vector_stores import MetadataFilter, MetadataFilters, ExactMatchFilter, FilterOperator
7
from llama_index.core.query_engine import RetrieverQueryEngine
8
from llama_index.embeddings.openai import OpenAIEmbedding
9
from llama_index.llms.openai import OpenAI
10
from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch

1
os.environ["OPENAI_API_KEY"] = getpass.getpass("OpenAI API Key:")
2
ATLAS_CONNECTION_STRING = getpass.getpass("MongoDB Atlas SRV Connection String:")

1
Settings.llm = OpenAI()
2
Settings.embed_model = OpenAIEmbedding(model="text-embedding-ada-002")

1
sample_data = SimpleDirectoryReader(input_files=["/content/tj_fall_faves_oct30.txt"]).load_data()
2
sample_data[0]

1
# connect to your Atlas cluster
2
mongo_client = pymongo.MongoClient(ATLAS_CONNECTION_STRING, appname = "devrel.showcase.tj_fall_faves")
3
4
5
# instantiate the vector store
6
atlas_vector_store = MongoDBAtlasVectorSearch(
7
   mongo_client,
8
   db_name = "tj_products",
9
   collection_name = "fall_faves",
10
   vector_index_name = "vector_index"
11
)
12
vector_store_context = StorageContext.from_defaults(vector_store=atlas_vector_store)

1
vector_store_index = VectorStoreIndex.from_documents(
2
  sample_data, storage_context=vector_store_context, show_progress=True
3
)

1
# Specify the collection for which to create the index
2
collection = mongo_client["tj_products"]["fall_faves"]
3
4
5
# Create your index model, then create the search index
6
search_index_model = SearchIndexModel(
7
 definition={
8
   "fields": [
9
     {
10
       "type": "vector",
11
       "path": "embedding",
12
       "numDimensions": 1536,
13
       "similarity": "cosine"
14
     },
15
     {
16
       "type": "filter",
17
       "path": "metadata.page_label"
18
     }
19
   ]
20
 },
21
 name="vector_index",
22
 type="vectorSearch",
23
)
24
25
26
collection.create_search_index(model=search_index_model)

1
vector_store_retriever = VectorIndexRetriever(index=vector_store_index, similarity_top_k=5)
2
3
4
query_engine = RetrieverQueryEngine(retriever=vector_store_retriever)
5
6
7
response = query_engine.query('Which plant items are available right now? Please provide prices')
8
9
10
print(response)

1
Mum Fleurettes are available for $4.99 and Assorted Mum Plants are available for $6.99.

1
# llamaindex chat engine
2
chat_engine = vector_store_index.as_chat_engine(
3
   chat_mode="condense_question", streaming=True
4
)

1
while True:
2
   # ask question
3
   question = input("Ask away! Type 'exit' to quit >>> ")
4
  
5
   # exit to quit
6
   if question == 'exit':
7
       print("Exiting chat. Have a happy fall!")
8
       break
9
10
11
   print("\n")

1
   # llamaindex ask
2
   response_stream = chat_engine.stream_chat(question)
3
  
4
   # llamaindex print
5
   response_stream.print_response_stream()
6
   print("\n")

1
Ask away! Type 'exit' to quit >>> hi! i am planning a fall party
2
3
4
Consider including a variety of fall-themed food and beverages such as pumpkin pie, apple cider donuts, maple-flavored fudge, pumpkin spiced cookies, and harvest blend herbal tea to create a festive atmosphere for your fall party. Additionally, you could incorporate seasonal decorations like cinnamon brooms, scented candles, and mum plants to enhance the autumn ambiance.
5
6
Ask away! Type 'exit' to quit >>> i want to make a turkey, which three sides with prices and reasonings will be best
7
8
9
The best three side dishes to serve with turkey at a fall party would be Cut Butternut Squash, Brussels Sprouts, and Cornbread Stuffing. Cut Butternut Squash and Brussels Sprouts are reasonably priced at $3.99 and $4.99 respectively, offering a balance of flavors and textures that complement the turkey well. Cornbread Stuffing, priced at $5.99, adds a traditional touch to the meal and enhances the overall fall-themed dining experience.
10
11
Ask away! Type 'exit' to quit >>> which drinks should i serve? i want something caffinated 
12
13
14
Harvest Blend Herbal Tea and Autumn Maple Coffee would be ideal caffeinated drinks to serve at a fall party to complement the autumn-themed food and create a festive atmosphere.
15
16
Ask away! Type 'exit' to quit >>> what are the prices of these drinks
17
18
19
$2.49 for Harvest Blend Herbal Tea and $8.99 for Autumn Maple Coffee.
20
21
Ask away! Type 'exit' to quit >>> which decor should i use? i want my home to smell nice
22
23
24
Cinnamon Whisk, Cinnamon Broom, Orange & Spice Scented Candle & Room Spritz
25
26
Ask away! Type 'exit' to quit >>> what are the prices?
27
28
29
$5.99, $1.29, $4.99
30
31
Ask away! Type 'exit' to quit >>> exit
32
Exiting chat. Have a happy fall!