Pesquisa semântica com Jina Embeddings v2 e MongoDB Atlas

1
pip install pymongo

1
from pymongo.mongo_client import MongoClient
2
3
client = MongoClient("<URL from above>")

1
db = client.sample_airbnb
2
collection = db.listingsAndReviews

1
print(collection.find_one())

1
{'_id': '10006546',
2
 'listing_url': 'https://www.airbnb.com/rooms/10006546',
3
 'name': 'Ribeira Charming Duplex',
4
 'summary': 'Fantastic duplex apartment with three bedrooms, located in the historic 
5
						area of Porto, Ribeira (Cube) - UNESCO World Heritage Site. Centenary 
6
						building fully rehabilitated, without losing their original character.',
7
 'space': 'Privileged views of the Douro River and Ribeira square, our apartment offers 
8
					the perfect conditions to discover the history and the charm of Porto. 
9
					Apartment comfortable, charming, romantic and cozy in the heart of Ribeira. 
10
					Within walking distance of all the most emblematic places of the city of Porto. 
11
					The apartment is fully equipped to host 8 people, with cooker, oven, washing 
12
					machine, dishwasher, microwave, coffee machine (Nespresso) and kettle. The 
13
					apartment is located in a very typical area of the city that allows to cross 
14
					with the most picturesque population of the city, welcoming, genuine and happy 
15
					people that fills the streets with his outspoken speech and contagious with 
16
					your sincere generosity, wrapped in a only parochial spirit.',
17
 'description': 'Fantastic duplex apartment with three bedrooms, located in the historic 
18
								area of Porto, Ribeira (Cube) - UNESCO World Heritage Site. Centenary 
19
								building fully rehabilitated, without losing their original character. 
20
								Privileged views of the Douro River and Ribeira square, our apartment 
21
								offers the perfect conditions to discover the history and the charm of 
22
								Porto. Apartment comfortable, charming, romantic and cozy in the heart of 
23
								Ribeira. Within walking distance of all the most emblematic places of the 
24
								city of Porto. The apartment is fully equipped to host 8 people, with 
25
								cooker, oven, washing machine, dishwasher, microwave, coffee machine 
26
								(Nespresso) and kettle. The apartment is located in a very typical area 
27
								of the city that allows to cross with the most picturesque population of 
28
								the city, welcoming, genuine and happy people that fills the streets with 
29
								his outspoken speech and contagious with your sincere generosity, wrapped 
30
								in a only parochial spirit. We are always available to help guests',
31
...
32
}

1
import requests
2
3
url = 'https://api.jina.ai/v1/embeddings'
4
5
headers = {
6
  'Content-Type': 'application/json',
7
  'Authorization': 'Bearer <insert your API key here>'
8
}
9
10
data = {
11
  'input': ["Your text string goes here"],
12
  'model': 'jina-embeddings-v2-base-en'
13
}
14
15
response = requests.post(url, headers=headers, json=data)
16
17
print(response.content)

1
b'{"model":"jina-embeddings-v2-base-en","object":"list","usage":{"total_tokens":14,
2
"prompt_tokens":14},"data":[{"object":"embedding","index":0,"embedding":[-0.14528547,
3
-1.0152762,1.3449358,0.48228237,-0.6381836,0.25765118,0.1794826,-0.5094953,0.5967494,
4
...,
5
-0.30768695,0.34024483,-0.5897042,0.058436804,0.38593403,-0.7729841,-0.6259417]}]}'

1
import requests
2
from pymongo.mongo_client import MongoClient
3
4
jinaai_token = "<your Jina token here>"
5
mongo_url = "<your MongoDB Atlas database URL>"
6
embedding_url = "https://api.jina.ai/v1/embeddings"

1
client = MongoClient(mongo_url)
2
db = client.sample_airbnb

1
def generate_embeddings(texts):
2
    payload = {"input": texts, 
3
               "model": "jina-embeddings-v2-base-en"}
4
    try:
5
        response = requests.post(
6
            embedding_url,
7
            headers={"Authorization": f"Bearer {jinaai_token}"},
8
            json=payload
9
        )
10
    except Exception as e:
11
        raise ValueError(f"Error in calling embedding API: {e}/nInput: {texts}")
12
    if response.status_code != 200:
13
        raise ValueError(f"Error in embedding service {response.status_code}: {response.text}, {texts}")
14
    embeddings = [d["embedding"] for d in response.json()["data"]]
15
    return embeddings

1
def index():
2
    collection = db.listingsAndReviews
3
    docs_to_encode = collection.find({ "embedding_summary" : { "$exists" : False } }).limit(30)
4
    for i, doc in enumerate(docs_to_encode):
5
        if i and i%5==0:
6
            print("Finished embedding", i, "documents")
7
        try:
8
            embedding_summary, embedding_description = generate_embeddings([doc["summary"], doc["description"]])
9
        except Exception as e:
10
            print("Error in embedding", doc["_id"], e)
11
            continue
12
        doc["embedding_summary"] = embedding_summary
13
        doc["embedding_description"] = embedding_description
14
        collection.replace_one({'_id': doc['_id']}, doc)

1
index()

1
{
2
  "mappings": {
3
    "dynamic": true,
4
    "fields": {
5
      "embedding_description": {
6
        "dimensions": 768,
7
        "similarity": "dotProduct",
8
        "type": "knnVector"
9
      },
10
      "embedding_summary": {
11
        "dimensions": 768,
12
        "similarity": "dotProduct",
13
        "type": "knnVector"
14
      }
15
    }
16
  }
17
}

1
def search(query):
2
		query_embedding = generate_embeddings([query])[0]
3
    results = db.listingsAndReviews.aggregate([
4
        {
5
            '$search': {
6
                "index": "listings_comments_semantic_search",
7
                "knnBeta": {
8
                    "vector": query_embedding,
9
                    "k": 3,
10
                    "path": ["embedding_summary", "embedding_description"]
11
                }
12
            }
13
        }
14
    ])
15
    for document in results:
16
        print(f'Listing Name: {document["name"]}\nSummary: {document["name"]}\nDescription: {document["description"]}\n\n')

1
search("an amazing view and close to amenities")

1
Listing Name: Rented Room
2
Summary: Rented Room
3
Description: Beautiful room and with a great location in the city of Rio de Janeiro
4
5
6
Listing Name: Spacious and well located apartment
7
Summary: Spacious and well located apartment
8
Description: Enjoy Porto in a spacious, airy and bright apartment, fully equipped, in a 
9
building with lift, located in a region full of cafes and restaurants, close to the subway 
10
and close to the best places of the city. The apartment offers total comfort for those 
11
who, besides wanting to enjoy the many attractions of the city, also like to relax and 
12
feel at home, All airy and bright, with a large living room, fully equipped kitchen, and a 
13
delightful balcony, which in the summer refreshes and in the winter protects from the cold 
14
and rain, accommodating up to six people very well. It has 40-inch interactive TV, internet
15
and high-quality wi-fi, and for those who want to work a little, it offers a studio with a 
16
good desk and an inspiring view. The apartment is all available to guests. I leave my guests
17
at ease, but I am available whenever they need me. It is a typical neighborhood of Porto, 
18
where you have silence and tranquility, little traffic, no noise, but everything at hand: 
19
good restaurants and c
20
21
22
Listing Name: Panoramic Ocean View Studio in  Quiet Setting
23
Summary: Panoramic Ocean View Studio in  Quiet Setting
24
Description: Luxury studio unit is located in a family-oriented neighborhood that lets you 
25
experience Hawaii like a local! with tranquility and serenity, while in close proximity to 
26
beaches and restaurants! The unit is surrounded by lush tropical vegetation! High-speed 
27
Wi-Fi available in the unit!! A large, private patio (lanai) with fantastic ocean views is 
28
completely under roof and is part of the studio unit. It's a great space for eating outdoors
29
or relaxing, while checking our the surfing action. This patio is like a living room 
30
without walls, with only a roof with lots and lots of skylights!!! We provide Wi-Fi and 
31
beach towels! The studio is detached from the main house, which has long-term tenants 
32
upstairs and downstairs. The lower yard and the front yard are assigned to those tenants, 
33
not the studio guests. The studio has exclusive use of its large (600 sqft) patio - under 
34
roof! Check-in and check-out times other than the ones listed, are by request only and an 
35
additional charges may apply; 
36
37
38
Listing Name: GOLF ROYAL RESIDENCE SUİTES(2+1)-2
39
Summary: GOLF ROYAL RESIDENCE SUİTES(2+1)-2
40
Description: A BIG BED ROOM WITH A BIG SALOON INCLUDING A NICE BALAKON TO HAVE SOME FRESH 
41
AIR . OUR RESIDENCE SITUATED AT THE CENTRE OF THE IMPORTANT MARKETS SUCH AS NİŞANTAŞİ,
42
OSMANBEY AND TAKSIM SQUARE,
43
44
45
Listing Name: DOUBLE ROOM for 1 or 2 ppl
46
Summary: DOUBLE ROOM for 1 or 2 ppl
47
Description: 10m2 with interior balkony kitchen, bathroom small but clean and modern metro
48
in front of the building 7min walk to Sagrada Familia, 2min walk TO amazing Gaudi Hospital
49
Sant Pau SAME PRICE FOR 1 OR 2 PPL-15E All flat for your use, terrace, huge TV.