Terraformando AI fluxos de trabalho de IA: RAG RAG com MongoDB Atlas e Spring AI AI

1
variable "atlas_org_id" {
2
  description = "MongoDB Atlas Organization ID"
3
  type        = string
4
}
5
6
variable "public_key" {
7
  description = "Public API key for MongoDB Atlas"
8
  type        = string
9
}
10
11
variable "private_key" {
12
  description = "Private API key for MongoDB Atlas"
13
  type        = string
14
}
15
16
variable "cluster_name" {
17
  description = "Name of the MongoDB Atlas cluster"
18
  type        = string
19
  default     = "RagCluster"
20
}
21
22
variable "project_name" {
23
  description = "Name of the MongoDB Atlas project"
24
  type        = string
25
  default     = "RAGProject"
26
}
27
28
variable "db_username" {
29
  description = "MongoDB database username"
30
  type        = string
31
}
32
33
variable "db_password" {
34
  description = "MongoDB database password"
35
  type        = string
36
}
37
38
variable "ip_address" {
39
  description = "IP address to whitelist"
40
  type        = string
41
}

1
terraform {
2
  required_providers {
3
    mongodbatlas = {
4
      source  = "mongodb/mongodbatlas"
5
    }
6
  }
7
  required_version = ">= 0.13"
8
}

1
provider "mongodbatlas" {
2
  public_key  = var.public_key
3
  private_key = var.private_key
4
}

1
resource "mongodbatlas_project" "rag_project" {
2
  name   = var.project_name
3
  org_id = var.atlas_org_id
4
}

1
resource "mongodbatlas_advanced_cluster" "rag_cluster" {
2
  project_id   = mongodbatlas_project.rag_project.id
3
  name         = var.cluster_name
4
  cluster_type = "REPLICASET"
5
6
  replication_specs {
7
    region_configs {
8
      electable_specs {
9
        instance_size = "M10"
10
        node_count    = 3
11
      }
12
      provider_name = "AWS"
13
      region_name   = "EU_WEST_1"
14
      priority      = 7
15
    }
16
  }
17
}

1
resource "mongodbatlas_project_ip_access_list" "ip_list" {
2
  project_id = mongodbatlas_project.rag_project.id
3
  ip_address = var.ip_address
4
}

1
resource "mongodbatlas_database_user" "db_user" {
2
  username           = var.db_username
3
  password           = var.db_password
4
  project_id         = mongodbatlas_project.rag_project.id
5
  auth_database_name = "admin"
6
7
  roles {
8
    role_name     = "readWrite"
9
    database_name = "rag"
10
  }
11
}

1
export TF_VAR_atlas_org_id="$MONGODB_ORG_ID"
2
export TF_VAR_public_key="$MONGODB_PUBLIC_KEY"
3
export TF_VAR_private_key="$MONGODB_PRIVATE_KEY"
4
export TF_VAR_db_username="$MONGODB_USER"
5
export TF_VAR_db_password="$MONGODB_PASSWORD"
6
export TF_VAR_ip_address="$IP_ADDRESS"

1
<dependencies>
2
    <dependency>
3
        <groupId>org.springframework.boot</groupId>
4
        <artifactId>spring-boot-starter-web</artifactId>
5
    </dependency>
6
7
    <dependency>
8
        <groupId>org.springframework.data</groupId>
9
        <artifactId>spring-data-mongodb</artifactId>
10
    </dependency>
11
12
    <dependency>
13
        <groupId>org.springframework.ai</groupId>
14
        <artifactId>spring-ai-openai</artifactId>
15
        <version>1.0.0-SNAPSHOT</version>
16
    </dependency>
17
</dependencies>

1
spring.application.name=RagApp
2
3
# OpenAI API key
4
spring.ai.openai.api-key=${OPENAI_API_KEY}
5
spring.ai.openai.chat.options.model=gpt-4
6
7
# MongoDB Atlas URI and Database
8
spring.data.mongodb.uri=${MONGO_URI}
9
spring.data.mongodb.database=rag
10
11
spring.ai.vectorstore.mongodb.initialize-schema=false

1
import org.springframework.ai.embedding.EmbeddingModel;
2
import org.springframework.ai.openai.OpenAiEmbeddingModel;
3
import org.springframework.ai.openai.api.OpenAiApi;
4
import org.springframework.ai.vectorstore.MongoDBAtlasVectorStore;
5
import org.springframework.ai.vectorstore.VectorStore;
6
import org.springframework.beans.factory.annotation.Value;
7
import org.springframework.context.annotation.Bean;
8
import org.springframework.context.annotation.Configuration;
9
import org.springframework.data.mongodb.core.MongoTemplate;
10
11
@Configuration
12
public class Config {
13
14
    @Value("${spring.ai.openai.api-key}")
15
    private String openAiKey;
16
17
    @Bean
18
    public EmbeddingModel embeddingModel() {
19
        return new OpenAiEmbeddingModel(new OpenAiApi(openAiKey));
20
    }
21
22
    @Bean
23
    public VectorStore mongodbVectorStore(MongoTemplate mongoTemplate, EmbeddingModel embeddingModel) {
24
        return new MongoDBAtlasVectorStore(mongoTemplate, embeddingModel,
25
                MongoDBAtlasVectorStore.MongoDBVectorStoreConfig.builder().build(), true);
26
    }
27
}

1
package com.mongodb.RagApp.service;
2
3
import com.fasterxml.jackson.databind
4
5
.ObjectMapper;
6
import org.springframework.ai.document.Document;
7
import org.springframework.ai.vectorstore.VectorStore;
8
import org.springframework.beans.factory.annotation.Autowired;
9
import org.springframework.core.io.ClassPathResource;
10
import org.springframework.stereotype.Service;
11
12
import java.io.BufferedReader;
13
import java.io.InputStream;
14
import java.io.InputStreamReader;
15
import java.util.ArrayList;
16
import java.util.List;
17
import java.util.Map;
18
19
@Service
20
public class DocsLoaderService {
21
    private static final int MAX_TOKENS_PER_CHUNK = 2000; 
22
    private final VectorStore vectorStore;
23
    private final ObjectMapper objectMapper;
24
25
    @Autowired
26
    public DocsLoaderService(VectorStore vectorStore, ObjectMapper objectMapper) {
27
        this.vectorStore = vectorStore;
28
        this.objectMapper = objectMapper;
29
    }
30
31
    public String loadDocs() {
32
        try (InputStream inputStream = new ClassPathResource("docs/devcenter-content-snapshot.json").getInputStream();
33
             BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
34
35
            List<Document> documents = new ArrayList<>();
36
            String line;
37
            while ((line = reader.readLine()) != null) {
38
                Map<String, Object> jsonDoc = objectMapper.readValue(line, Map.class);
39
                String content = (String) jsonDoc.get("body");
40
41
                List<String> chunks = splitIntoChunks(content, MAX_TOKENS_PER_CHUNK);
42
                for (String chunk : chunks) {
43
                    Document document = createDocument(jsonDoc, chunk);
44
                    documents.add(document);
45
                }
46
47
                if (documents.size() >= 100) {
48
                    vectorStore.add(documents);
49
                    documents.clear();
50
                }
51
            }
52
53
            if (!documents.isEmpty()) {
54
                vectorStore.add(documents);
55
            }
56
57
            return "All documents added successfully!";
58
        } catch (Exception e) {
59
            return "Error while adding documents: " + e.getMessage();
60
        }
61
    }
62
63
    private Document createDocument(Map<String, Object> jsonMap, String content) {
64
        Map<String, Object> metadata = (Map<String, Object>) jsonMap.get("metadata");
65
        metadata.putIfAbsent("sourceName", jsonMap.get("sourceName"));
66
        metadata.putIfAbsent("url", jsonMap.get("url"));
67
68
        return new Document(content, metadata);
69
    }
70
71
    private List<String> splitIntoChunks(String content, int maxTokens) {
72
        List<String> chunks = new ArrayList<>();
73
        String[] words = content.split("\\s+");
74
        StringBuilder chunk = new StringBuilder();
75
        int tokenCount = 0;
76
77
        for (String word : words) {
78
            int wordTokens = word.length() / 4;
79
            if (tokenCount + wordTokens > maxTokens) {
80
                chunks.add(chunk.toString());
81
                chunk.setLength(0);
82
                tokenCount = 0;
83
            }
84
            chunk.append(word).append(" ");
85
            tokenCount += wordTokens;
86
        }
87
88
        if (chunk.length() > 0) {
89
            chunks.add(chunk.toString());
90
        }
91
92
        return chunks;
93
    }
94
}

1
import org.springframework.ai.chat.client.ChatClient;
2
import org.springframework.ai.chat.client.advisor.QuestionAnswerAdvisor;
3
import org.springframework.ai.vectorstore.SearchRequest;
4
import org.springframework.ai.vectorstore.VectorStore;
5
import org.springframework.web.bind.annotation.CrossOrigin;
6
import org.springframework.web.bind.annotation.GetMapping;
7
import org.springframework.web.bind.annotation.RequestParam;
8
import org.springframework.web.bind.annotation.RestController;
9
10
@RestController
11
public class RagController {
12
    private final ChatClient chatClient;
13
14
    public RagController(ChatClient.Builder builder, VectorStore vectorStore) {
15
        this.chatClient = builder
16
                .defaultAdvisors(new QuestionAnswerAdvisor(vectorStore, SearchRequest.defaults()))
17
                .build();
18
    }
19
20
    @GetMapping("/question")
21
    public String question(@RequestParam(value = "message", defaultValue = "What is RAG?") String message) {
22
        return chatClient.prompt()
23
                .user(message)
24
                .call()
25
                .content();
26
    }
27
}

1
curl http://localhost:8080/api/docs/load

1
spring.ai.vectorstore.mongodb.initialize-schema=true

1
resource "mongodbatlas_search_index" "vector_search" {
2
  name   = "search-index"  
3
  project_id = mongodbatlas_project.rag_project.id
4
  cluster_name = mongodbatlas_advanced_cluster.rag_cluster.name
5
  type = "vectorSearch"
6
  database = "rag"
7
  collection_name = "vector_store"
8
  fields = <<-EOF
9
    [{
10
        "type": "vector",
11
        "path": "embedding",
12
        "numDimensions": 1536,
13
        "similarity": "cosine"
14
    }]
15
    EOF
16
}

1
terraform init
2
terraform apply

1
curl http://localhost:8080/question?message=How%20to%20analyze%20time-series%20data%20with%20Python%20and%20MongoDB?