Além do básico: aprimorando a API Kotlin Ktor com pesquisa vetorial

1
exports = async function(changeEvent) {
2
    const doc = changeEvent.fullDocument;
3
4
    const url = 'https://api-inference.huggingface.co/pipeline/feature-extraction/sentence-transformers/all-MiniLM-L6-v2';
5
    const hf_key = context.values.get("HF_value");
6
    try {
7
        console.log(`Processing document with id: ${doc._id}`);
8
9
        let response = await context.http.post({
10
            url: url,
11
            headers: {
12
                'Authorization': [`Bearer ${hf_key}`],
13
                'Content-Type': ['application/json']
14
            },
15
16
            body: JSON.stringify({               
17
                inputs: [doc.description]               
18
            })
19
        });
20
21
        let responseData = EJSON.parse(response.body.text());
22
23
        if(response.statusCode === 200) {
24
            console.log("Successfully received embedding.");
25
26
            const embedding = responseData[0];
27
28
            const collection = context.services.get("Cluster0").db("my_database").collection("exercises");
29
30
            const result = await collection.updateOne(
31
                { _id: doc._id },
32
                { $set: { descEmbedding: embedding }}
33
            );
34
35
            if(result.modifiedCount === 1) {
36
                console.log("Successfully updated the document.");
37
            } else {
38
                console.log("Failed to update the document.");
39
            }
40
        } else {
41
            console.log(`Failed to receive embedding. Status code: ${response.statusCode} plus ${response}`);
42
        }
43
44
    } catch(err) {
45
        console.error(err);
46
    }
47
};

1
 .\mongoimport mongodb+srv://<user>:<password>@cluster0.xpto.mongodb.net/my_database --collection exercises --jsonArray .\exercises.json

1
//Client
2
implementation("io.ktor:ktor-client-core:$ktor_version")
3
implementation("io.ktor:ktor-client-cio:$ktor_version")

1
package com.mongodb.domain.ports
2
3
import com.mongodb.domain.entity.Exercises
4
5
interface ExercisesRepository {
6
    suspend fun findSimilarExercises(embedding: List<Double>): List<Exercises>
7
}

1
package com.mongodb.application.response
2
data class ExercisesResponse(
3
    val exerciseNumber: Int,
4
    val bodyPart: String,
5
    val type: String,
6
    val description: String,
7
    val title: String
8
)

1
package com.mongodb.domain.entity
2
3
4
import com.mongodb.application.response.ExercisesResponse
5
import org.bson.codecs.pojo.annotations.BsonId
6
import org.bson.types.ObjectId
7
8
9
data class Exercises(
10
    @BsonId
11
    val id: ObjectId,
12
    val exerciseNumber: Int,
13
    val title: String,
14
    val description: String,
15
    val type: String,
16
    val bodyPart: String,
17
    val equipment: String,
18
    val level: String,
19
    val rating: Double,
20
    val ratingDesc: String,
21
    val descEmbedding: List<Double>
22
){
23
    fun toResponse() = ExercisesResponse(
24
        exerciseNumber = exerciseNumber,
25
        title = title,
26
        description = description,
27
        bodyPart = bodyPart,
28
        type = type
29
    )
30
}

1
package com.mongodb.infrastructure
2
3
import com.mongodb.domain.entity.Exercises
4
import com.mongodb.domain.ports.ExercisesRepository
5
import com.mongodb.kotlin.client.coroutine.MongoDatabase
6
import kotlinx.coroutines.flow.toList
7
import org.bson.Document
8
9
class ExercisesRepositoryImpl(
10
    private val mongoDatabase: MongoDatabase
11
) : ExercisesRepository {
12
13
    companion object {
14
        const val EXERCISES_COLLECTION = "exercises"
15
    }
16
17
    override suspend fun findSimilarExercises(embedding: List<Double>): List<Exercises> {
18
        val result =
19
        mongoDatabase.getCollection<Exercises>(EXERCISES_COLLECTION).withDocumentClass<Exercises>().aggregate(
20
            listOf(
21
                Document(
22
                    "\$vectorSearch",
23
                    Document("queryVector", embedding)
24
                    .append("path", "descEmbedding")
25
                    .append("numCandidates", 3L)
26
                    .append("index", "vector_index")
27
                    .append("limit", 3L)
28
                )
29
            )
30
        )
31
32
        return result.toList()
33
    }
34
}

1
package com.mongodb.application.routes
2
3
import com.mongodb.application.request.SentenceRequest
4
import com.mongodb.domain.ports.ExercisesRepository
5
import com.mongodb.huggingFaceApiUrl
6
import io.ktor.client.*
7
import io.ktor.client.call.*
8
import io.ktor.client.engine.cio.*
9
import io.ktor.client.request.*
10
import io.ktor.client.statement.*
11
import io.ktor.http.*
12
import io.ktor.http.content.*
13
import io.ktor.server.application.*
14
import io.ktor.server.request.*
15
import io.ktor.server.response.*
16
import io.ktor.server.routing.*
17
import org.koin.ktor.ext.inject
18
19
fun Route.exercisesRoutes() {
20
   val repository by inject<ExercisesRepository>()
21
22
   route("/exercises/processRequest") {
23
       post {
24
           val sentence = call.receive<SentenceRequest>()
25
26
           val response = requestSentenceTransform(sentence.input, call.huggingFaceApiUrl())
27
28
           if (response.status.isSuccess()) {
29
               val embedding = sentence.convertResponse(response.body())
30
               val similarDocuments = repository.findSimilarExercises(embedding)
31
32
               call.respond(HttpStatusCode.Accepted, similarDocuments.map { it.toResponse() })
33
           }
34
       }
35
   }
36
}
37
38
suspend fun requestSentenceTransform(input: String, huggingFaceURL: String): HttpResponse {
39
40
   println(huggingFaceURL)
41
42
   return HttpClient(CIO).use { client ->
43
44
       val response = client.post(huggingFaceURL) {
45
           val content = TextContent(input, ContentType.Text.Plain)
46
           setBody(content)
47
       }
48
49
       response
50
   }
51
}

1
package com.mongodb.application.request
2
3
data class SentenceRequest(
4
   val input: String
5
)
6
{
7
   fun convertResponse(body: String) =
8
       body
9
           .replace("[", "")
10
           .replace("]", "")
11
           .split(",")
12
           .map { it.trim().toDouble() }
13
}

1
package com.mongodb
2
3
import com.mongodb.application.routes.exercisesRoutes
4
import com.mongodb.application.routes.fitnessRoutes
5
import com.mongodb.domain.ports.ExercisesRepository
6
import com.mongodb.domain.ports.FitnessRepository
7
import com.mongodb.infrastructure.ExercisesRepositoryImpl
8
import com.mongodb.infrastructure.FitnessRepositoryImpl
9
import com.mongodb.kotlin.client.coroutine.MongoClient
10
import io.ktor.serialization.gson.*
11
import io.ktor.server.application.*
12
import io.ktor.server.plugins.contentnegotiation.*
13
import io.ktor.server.plugins.swagger.*
14
import io.ktor.server.routing.*
15
import io.ktor.server.tomcat.*
16
import org.koin.dsl.module
17
import org.koin.ktor.plugin.Koin
18
import org.koin.logger.slf4jLogger
19
20
fun main(args: Array<String>): Unit = EngineMain.main(args)
21
fun Application.module() {
22
23
   install(ContentNegotiation) {
24
       gson {
25
       }
26
   }
27
28
	// Other code..
29
  
30
   routing {
31
      // Other code..
32
     
33
      exercisesRoutes() 
34
   }
35
}
36
37
fun ApplicationCall.huggingFaceApiUrl(): String {
38
   return application.environment.config.propertyOrNull("ktor.huggingface.api.url")?.getString()
39
       ?: throw RuntimeException("Failed to access Hugging Face API base URL.")
40
41
}

1
ktor {
2
  
3
    // Other code ..
4
5
   huggingface {
6
       api {
7
           url = "https://api-inference.huggingface.co/pipeline/feature-extraction/sentence-transformers/all-MiniLM-L6-v2"
8
       }
9
   }
10
}

1
{
2
  "fields": [
3
    {
4
      "type": "vector",
5
      "path": "descEmbedding",
6
      "numDimensions": 384,
7
      "similarity": "euclidean"
8
    }
9
  ]
10
}

1
curl --location 'http://localhost:8081/exercises/processRequest' \
2
--header 'Content-Type: application/json' \
3
--data '{
4
    "input": "I need an exercise for my shoulders and to lose my belly fat"
5
}'