PyMongoArrow: fazendo a ponte entre o MongoDB e seu aplicativo de análise de dados

1
$ python -m pip install jupyter pymongoarrow 'pymongo[srv]' pandas

1
export MDB_URI=mongodb+srv://USERNAME:PASSWORD@CLUSTERID.azure.mongodb.net/sample_weatherdata?retryWrites=true&w=majority

1
{'_id': ObjectId('5553a998e4b02cf7151190bf'),
2
  'st': 'x+49700-055900',
3
  'ts': datetime.datetime(1984, 3, 5, 15, 0),
4
  'position': {'type': 'Point', 'coordinates': [-55.9, 49.7]},
5
  'elevation': 9999,
6
  'callLetters': 'SCGB',
7
  'qualityControlProcess': 'V020',
8
  'dataSource': '4',
9
  'type': 'FM-13',
10
  'airTemperature': {'value': -5.1, 'quality': '1'},
11
  'dewPoint': {'value': 999.9, 'quality': '9'},
12
  'pressure': {'value': 1020.8, 'quality': '1'},
13
  'wind': {'direction': {'angle': 100, 'quality': '1'},
14
   'type': 'N',
15
   'speed': {'rate': 3.1, 'quality': '1'}},
16
  'visibility': {'distance': {'value': 20000, 'quality': '1'},
17
   'variability': {'value': 'N', 'quality': '9'}},
18
  'skyCondition': {'ceilingHeight': {'value': 22000,
19
    'quality': '1',
20
    'determination': 'C'},
21
   'cavok': 'N'},
22
  'sections': ['AG1', 'AY1', 'GF1', 'MD1', 'MW1'],
23
  'precipitationEstimatedObservation': {'discrepancy': '2',
24
   'estimatedWaterDepth': 0},
25
  'pastWeatherObservationManual': [{'atmosphericCondition': {'value': '0',
26
     'quality': '1'},
27
    'period': {'value': 3, 'quality': '1'}}],
28
  'skyConditionObservation': {'totalCoverage': {'value': '01',
29
    'opaque': '99',
30
    'quality': '1'},
31
   'lowestCloudCoverage': {'value': '01', 'quality': '1'},
32
   'lowCloudGenus': {'value': '01', 'quality': '1'},
33
   'lowestCloudBaseHeight': {'value': 800, 'quality': '1'},
34
   'midCloudGenus': {'value': '00', 'quality': '1'},
35
   'highCloudGenus': {'value': '00', 'quality': '1'}},
36
  'atmosphericPressureChange': {'tendency': {'code': '8', 'quality': '1'},
37
   'quantity3Hours': {'value': 0.5, 'quality': '1'},
38
   'quantity24Hours': {'value': 99.9, 'quality': '9'}},
39
  'presentWeatherObservationManual': [{'condition': '02', 'quality': '1'}]}

1
import os
2
import pyarrow
3
import pymongo
4
import bson
5
import pymongoarrow.monkey
6
from pymongoarrow.api import Schema
7
8
MDB_URI = os.environ['MDB_URI']
9
10
11
# Add extra find_* methods to pymongo collection objects:
12
pymongoarrow.monkey.patch_all()
13
14
client = pymongo.MongoClient(MDB_URI)
15
database = client.get_default_database()
16
collection = database.get_collection("data")

1
collection.find_pandas_all(
2
    {},
3
    schema=Schema({
4
        'ts': pyarrow.timestamp('ms'),
5
    })
6
)

1
list(collection.aggregate([
2
    {'$match': {'_id': bson.ObjectId("5553a998e4b02cf7151190bf")}},
3
    {'$project': {
4
        'windDirection': '$wind.direction.angle',
5
        'windSpeed': '$wind.speed.rate',
6
    }}
7
]))
8
9
[{'_id': ObjectId('5553a998e4b02cf7151190bf'),
10
  'windDirection': 100,
11
  'windSpeed': 3.1}]

1
collection.aggregate_pandas_all([
2
        {'$project': {
3
            'windDirection': '$wind.direction.angle',
4
            'windSpeed': '$wind.speed.rate',
5
        }}
6
    ],
7
    schema=Schema({'windDirection': int, 'windSpeed': float})
8
)