Construindo uma base de conhecimento e gráficos de visualização para RAG com MongoDB

1
llm = ChatOpenAI(temperature=0, model_name="gpt-4-turbo",api_key="YOUR-OPENAI-KEY")
2
3
llm_transformer = LLMGraphTransformer(llm=llm)
4
text = """
5
Python was invented in the late 1980s by Guido van Rossum at Centrum Wiskunde & Informatica in the Netherlands as a successor to the ABC 
6
programming language, which was inspired by SETL capable of exception handling and interfacing with the Amoeba operating system. 
7
Its implementation began in December 1989. Python 2.0 was released on 16 October 2000, with many major new features such as list comprehensions, 
8
cycle-detecting garbage collection, reference counting, and Unicode support. Python 3.0, released on 3 December 2008, 
9
with many of its major features backported to Python 2.6.x and 2.7.x. Releases of Python 3 include the 2to3 utility, 
10
which automates the translation of Python 2 code to Python 3.
11
"""
12
documents = [Document(page_content=text)]
13
graph_documents = llm_transformer.convert_to_graph_documents(documents)
14
print(f"Nodes:{graph_documents[0].nodes}")
15
print(f"Relationships:{graph_documents[0].relationships}")

1
Nodes:[Node(id='Python', type='Programming_language'), Node(id='Guido Van Rossum', type='Person'), Node(id='Centrum Wiskunde & Informatica', type='Organization'), Node(id='Netherlands', type='Country'), Node(id='Abc Programming Language', type='Programming_language'), Node(id='Setl', type='Programming_language'), Node(id='Amoeba Operating System', type='Operating_system'), Node(id='Python 2.0', type='Software_version'), Node(id='Python 3.0', type='Software_version'), Node(id='2To3 Utility', type='Software_tool')]
2
Relationships:[Relationship(source=Node(id='Python', type='Programming_language'), target=Node(id='Guido Van Rossum', type='Person'), type='CREATED_BY'), Relationship(source=Node(id='Python', type='Programming_language'), target=Node(id='Centrum Wiskunde & Informatica', type='Organization'), type='DEVELOPED_AT'), Relationship(source=Node(id='Python', type='Programming_language'), target=Node(id='Netherlands', type='Country'), type='DEVELOPED_IN'), Relationship(source=Node(id='Python', type='Programming_language'), target=Node(id='Abc Programming Language', type='Programming_language'), type='SUCCESSOR_OF'), Relationship(source=Node(id='Abc Programming Language', type='Programming_language'), target=Node(id='Setl', type='Programming_language'), type='INSPIRED_BY'), Relationship(source=Node(id='Python', type='Programming_language'), target=Node(id='Amoeba Operating System', type='Operating_system'), type='INTERFACE_WITH'), Relationship(source=Node(id='Python 3.0', type='Software_version'), target=Node(id='Python 2.0', type='Software_version'), type='BACKPORTED_TO'), Relationship(source=Node(id='Python 3.0', type='Software_version'), target=Node(id='2To3 Utility', type='Software_tool'), type='INCLUDES')]

1
nodes = graph_documents[0].nodes
2
relationships = graph_documents[0].relationships
3
collections = set()
4
for node in nodes:
5
    collections.add(node.type)
6
print(collections)
7
try:
8
    uri = "MONGO-DB-URL"
9
    client = MongoClient(uri)
10
    database = client["generic_graph"]
11
    for collection in collections:
12
        database.create_collection(collection)
13
except Exception as e:
14
    print(e)
15
finally:
16
    client.close()

1
{'Programming_language', 'Software_tool', 'Person', 'Software_version', 'Organization', 'Country', 'Operating_system'}

1
 #Figuring out all relationship types per node types
2
node_relationship_types = {}
3
for node in nodes:
4
    #print(f'On Node {node.id}')
5
    node_relationship_types[node.id] = set()
6
    for rel in relationships:
7
        #print(f'Looking at Relationship for {rel.source.id}')
8
        if rel.source.id == node.id:
9
            node_relationship_types[node.id].add(rel.type)
10
print(node_relationship_types)

1
{'Python': {'DEVELOPED_IN', 'CREATED_BY', 'DEVELOPED_AT', 'SUCCESSOR_OF', 'INTERFACE_WITH'}, 'Guido Van Rossum': set(), 'Centrum Wiskunde & Informatica': set(), 'Netherlands': set(), 'Abc Programming Language': {'INSPIRED_BY'}, 'Setl': set(), 'Amoeba Operating System': set(), 'Python 2.0': set(), 'Python 3.0': {'BACKPORTED_TO'}, '2To3 Utility': {'TRANSLATES_TO'}}

1
mongo_documents = []
2
for node in nodes:
3
    document_dict = {}
4
    document_dict['id'] = node.id
5
    document_dict['type'] = node.type
6
    document_relations = node_relationship_types[node.id]
7
    for document_relation in document_relations:
8
        document_dict[document_relation] = []
9
    for rel in relationships:
10
        if rel.source.id == node.id:
11
            document_dict[rel.type].append(rel.target.id)
12
    mongo_documents.append(document_dict)
13
print(mongo_documents)

1
[{'id': 'Python', 'type': 'Programming_language', 'DEVELOPED_IN': ['Netherlands'], 'CREATED_BY': ['Guido Van Rossum'], 'DEVELOPED_AT': ['Centrum Wiskunde & Informatica'], 'SUCCESSOR_OF': ['Abc Programming Language'], 'INTERFACE_WITH': ['Amoeba Operating System']}, {'id': 'Guido Van Rossum', 'type': 'Person'}, {'id': 'Centrum Wiskunde & Informatica', 'type': 'Organization'}, {'id': 'Netherlands', 'type': 'Country'}, {'id': 'Abc Programming Language', 'type': 'Programming_language', 'INSPIRED_BY': ['Setl']}, {'id': 'Setl', 'type': 'Programming_language'}, {'id': 'Amoeba Operating System', 'type': 'Operating_system'}, {'id': 'Python 2.0', 'type': 'Software_version'}, {'id': 'Python 3.0', 'type': 'Software_version', 'BACKPORTED_TO': ['Python 2.0']}, {'id': '2To3 Utility', 'type': 'Software_tool', 'TRANSLATES_TO': ['Python 3.0']}]

1
try:
2
    uri = "YOUR-MONGO-URL"
3
    client = MongoClient(uri)
4
    database = client["generic_graph"]
5
    for mongo_document in mongo_documents:
6
        collection = database[mongo_document['type']]
7
        collection.insert_one(mongo_document)
8
except Exception as e:
9
    print(e)
10
finally:
11
    client.close()

1
[{'CREATED_BY': ['Guido Van Rossum'],
2
  'DEVELOPED_AT': ['Centrum Wiskunde & Informatica'],
3
  'DEVELOPED_IN': ['Netherlands'],
4
  'INTERFACE_WITH': ['Amoeba Operating System'],
5
  'SUCCESSOR_OF': ['Abc Programming Language'],
6
  'd3_edges': [{'linkName': 'CREATED_BY',
7
                'source': 'Python',
8
                'strength': 0.7,
9
                'target': 'Guido Van Rossum'},
10
               {'linkName': 'DEVELOPED_AT',
11
                'source': 'Python',
12
                'strength': 0.7,
13
                'target': 'Centrum Wiskunde & Informatica'},
14
               {'linkName': 'DEVELOPED_IN',
15
                'source': 'Python',
16
                'strength': 0.7,
17
                'target': 'Netherlands'},
18
               {'linkName': 'SUCCESSOR_OF',
19
                'source': 'Python',
20
                'strength': 0.7,
21
                'target': 'Abc Programming Language'},
22
               {'linkName': 'INTERFACE_WITH',
23
                'source': 'Python',
24
                'strength': 0.7,
25
                'target': 'Amoeba Operating System'}],
26
  'd3_source_node': {'group': 0, 'id': 'Python', 'label': 'Python', 'level': 1},
27
  'd3_target_nodes': [{'group': 1,
28
                       'id': 'Guido Van Rossum',
29
                       'label': 'Guido Van Rossum',
30
                       'level': 2},
31
                      {'group': 1,
32
                       'id': 'Centrum Wiskunde & Informatica',
33
                       'label': 'Centrum Wiskunde & Informatica',
34
                       'level': 2},
35
                      {'group': 1,
36
                       'id': 'Netherlands',
37
                       'label': 'Netherlands',
38
                       'level': 2},
39
                      {'group': 1,
40
                       'id': 'Abc Programming Language',
41
                       'label': 'Abc Programming Language',
42
                       'level': 2},
43
                      {'group': 1,
44
                       'id': 'Amoeba Operating System',
45
                       'label': 'Amoeba Operating System',
46
                       'level': 2}],
47
  'id': 'Python',
48
  'type': 'Programming_language'}]

1
mongo_documents = []
2
for node in nodes:
3
    document_dict = {}
4
    document_dict['id'] = node.id
5
    document_dict['type'] = node.type
6
    document_dict['d3_edges'] = []
7
    document_dict['d3_target_nodes'] = []
8
    document_dict['d3_source_node'] = {'id':node.id,'group':0,'level':1,'label':node.id}
9
    document_relations = node_relationship_types[node.id]
10
    for document_relation in document_relations:
11
        document_dict[document_relation] = []
12
    for rel in relationships:
13
        if rel.source.id == node.id:
14
            document_dict[rel.type].append(rel.target.id)
15
            document_dict['d3_target_nodes'].append({'id':rel.target.id,'group':1,'level':2,'label':rel.target.id})
16
            document_dict['d3_edges'].append({'source':node.id,'target':rel.target.id,'strength':0.7,'linkName':rel.type})
17
    mongo_documents.append(document_dict)
18
pprint(mongo_documents)

1
    Object {source: "X1", target: "X2", type: "suit"}
2
    Object {source: "X2", target: "X3", type: "resolved"}
3
    Object {source: "X3", target: "X4", type: "suit"}
4
    Object {source: "X4", target: "X1", type: "suit"}
5
    columns: Array(3) [
6
     0: "source"
7
     1: "target"
8
     2: "type"
9
    ]

1
     Object {source: "Napoleon", target: "Myriel", value: 1}
2
     Object {source: "Mlle.Baptistine", target: "Myriel", value: 8}
3
     Object {source: "Mme.Magloire", target: "Myriel", value: 10}
4
     Object {source: "Mme.Magloire", target: "Mlle.Baptistine", value: 6}
5
     Object {source: "Cravatte", target: "Myriel", value: 1}
6
     Object {source: "Count", target: "Myriel", value: 2}
7
     Object {source: "OldMan", target: "Myriel", value: 1}

1
    data = Object {
2
     name: "flare"
3
     children: Array(10) [
4
     0: Object {
5
     name: "analytics"
6
     children: Array(3) [
7
     0: Object {name: "cluster", children: Array(4)}
8
     1: Object {name: "graph", children: Array(5)}
9
     2: Object {name: "optimization", children: Array(1)}
10
    ]
11
    }
12
     1: Object {name: "animate", children: Array(12)}
13
     2: Object {name: "data", children: Array(7)}
14
     3: Object {name: "display", children: Array(4)}
15
     4: Object {
16
     name: "flex"
17
     children: Array(1) [
18
     0: Object {
19
    }
20
    ]
21
    }

1
{nodes = [{'id': 'Python', 'group': 0, 'level': 1, 'label': 'Python'}, {'id': 'Netherlands', 'group': 1, 'level': 2, 'label': 'Netherlands'}],
2
links = [{'source': 'Python', 'target': 'Netherlands', 'strength': 0.7}, {'source': 'Python', 'target': 'Guido Van Rossum', 'strength': 0.7}]}

1
try:
2
    uri = "YOUR-MONGO-URL"
3
    client = MongoClient(uri)
4
    database = client["generic_graph"]
5
    language_collection = database["Programming_language"]
6
    language_pipeline = [
7
    {
8
        '$graphLookup': {
9
            'from': 'Programming_language', 
10
            'startWith': '$SUCCESSOR_OF', 
11
            'connectFromField': 'id', 
12
            'connectToField': 'id', 
13
            'as': 'relations', 
14
            'maxDepth': 2
15
        }
16
    }, {
17
        '$unwind': {
18
            'path': '$relations', 
19
            'preserveNullAndEmptyArrays': False
20
        }
21
    }
22
    ]
23
    lang_aggCursor = language_collection.aggregate(language_pipeline)
24
    nodes=[]
25
    links=[]
26
    for document in lang_aggCursor:
27
        print(document)
28
        source_node_dict = {}
29
        source_node_dict['id'] = document.get('id')
30
        source_node_dict['group'] = 0
31
        source_node_dict['level'] = 1
32
        source_node_dict['label'] = document.get('id')
33
        nodes.append(source_node_dict)
34
        for key in document.keys():
35
            print(key)
36
            target_node_dict = {}
37
            link_dict = {}
38
            if key=='_id' or key=='id' or key=='type' or key=='SUCCESSOR_OF':
39
                continue
40
            elif key=='relations':
41
                target_node_dict['id']=document[key]['id']
42
                target_node_dict['group']=1
43
                target_node_dict['level']=2
44
                target_node_dict['label'] = document[key]['id']
45
                inspired_node_dict = {}
46
                inspired_node_dict['id'] = document[key]['INSPIRED_BY'][0]
47
                inspired_node_dict['group'] = 1
48
                inspired_node_dict['level'] = 2
49
                inspired_node_dict['label'] = document[key]['INSPIRED_BY'][0]
50
                link_dict['source'] = target_node_dict.get('id')
51
                link_dict['target'] = inspired_node_dict.get('id')
52
                link_dict['strength'] = 0.7
53
                link_dict['linkName'] = 'INSPIRED_BY'
54
                link_dict_2 = {}
55
                link_dict_2['source'] = source_node_dict.get('id')
56
                link_dict_2['target'] = target_node_dict.get('id')
57
                link_dict_2['strength'] = 0.7
58
                link_dict_2['linkName'] = 'SUCCESSOR_OF'
59
                nodes.append(target_node_dict)
60
                nodes.append(inspired_node_dict)
61
                links.append(link_dict)
62
                links.append(link_dict_2)
63
                continue
64
            else:
65
                target_node_dict['id'] = document[key][0]
66
                target_node_dict['group']=1
67
                target_node_dict['level']=2
68
                target_node_dict['label'] = document[key][0]
69
            link_dict['source'] = source_node_dict.get('id')
70
            link_dict['target'] = target_node_dict.get('id')
71
            link_dict['strength'] = 0.7
72
            link_dict['linkName'] = key
73
            nodes.append(target_node_dict)
74
            links.append(link_dict)
75
    print(nodes)
76
    print(links)
77
except Exception as e:
78
    print(e)
79
finally:
80
    client.close()
81
nodes_links = {"nodes": nodes,"links":links }
82
import json
83
with open("python-dependencies.json",'w') as f:
84
    json.dump(nodes_links,f,indent=1)

1
{nodes=[{'id': 'Python', 'group': 0, 'level': 1, 'label': 'Python'}, {'id': 'Netherlands', 'group': 1, 'level': 2, 'label': 'Netherlands'}, {'id': 'Guido Van Rossum', 'group': 1, 'level': 2, 'label': 'Guido Van Rossum'}, {'id': 'Centrum Wiskunde & Informatica', 'group': 1, 'level': 2, 'label': 'Centrum Wiskunde & Informatica'}, {'id': 'Amoeba Operating System', 'group': 1, 'level': 2, 'label': 'Amoeba Operating System'}, {'id': 'Abc Programming Language', 'group': 1, 'level': 2, 'label': 'Abc Programming Language'}, {'id': 'Setl', 'group': 1, 'level': 2, 'label': 'Setl'}],
2
links=[{'source': 'Python', 'target': 'Netherlands', 'strength': 0.7, 'linkName': 'DEVELOPED_IN'}, {'source': 'Python', 'target': 'Guido Van Rossum', 'strength': 0.7, 'linkName': 'CREATED_BY'}, {'source': 'Python', 'target': 'Centrum Wiskunde & Informatica', 'strength': 0.7, 'linkName': 'DEVELOPED_AT'}, {'source': 'Python', 'target': 'Amoeba Operating System', 'strength': 0.7, 'linkName': 'INTERFACE_WITH'}, {'source': 'Abc Programming Language', 'target': 'Setl', 'strength': 0.7, 'linkName': 'INSPIRED_BY'}, {'source': 'Python', 'target': 'Abc Programming Language', 'strength': 0.7, 'linkName': 'SUCCESSOR_OF'}]}

1
master_lookup_set = set()
2
nodes=[]
3
links=[]
4
try:
5
    uri = "YOUR-MONGODB-CLUSTER-URL"
6
    client = MongoClient(uri)
7
    database = client["embedded_graph_2"]
8
    collection = database["Programming_language"]
9
    cursor = collection.find({},{'_id':0,'id':1,'d3_edges':1,'d3_target_nodes':1,'d3_source_node':1})
10
    for document in cursor:
11
        print(document)
12
        if document['id'] not in master_lookup_set:
13
            master_lookup_set.add(document['id'])
14
            nodes.append(document['d3_source_node'])
15
            for link in document['d3_edges']:
16
                links.append(link)
17
                master_lookup_set.add(link['target'])
18
            for target_node in document['d3_target_nodes']:
19
                nodes.append(target_node)
20
        else:
21
            for link in document['d3_edges']:
22
                links.append(link)
23
except Exception as e:
24
    print(e)
25
finally:
26
    client.close()
27
28
nodes_links = {"nodes": nodes,"links":links }
29
import json
30
with open("python-dependencies_embedded_2.json",'w') as f:
31
    json.dump(nodes_links,f,indent=1)