Getting Started with Aggregation Pipelines in Python

1
{
2
   '_id': ObjectId('573a1392f29313caabcdb497'),
3
   'awards': {'nominations': 7,
4
               'text': 'Won 1 Oscar. Another 2 wins & 7 nominations.',
5
               'wins': 3},
6
   'cast': ['Janet Gaynor', 'Fredric March', 'Adolphe Menjou', 'May Robson'],
7
   'countries': ['USA'],
8
   'directors': ['William A. Wellman', 'Jack Conway'],
9
   'fullplot': 'Esther Blodgett is just another starry-eyed farm kid trying to '
10
               'break into the movies. Waitressing at a Hollywood party, she '
11
               'catches the eye of alcoholic star Norman Maine, is given a test, '
12
               'and is caught up in the Hollywood glamor machine (ruthlessly '
13
               'satirized). She and her idol Norman marry; but his career '
14
               'abruptly dwindles to nothing',
15
   'genres': ['Drama'],
16
   'imdb': {'id': 29606, 'rating': 7.7, 'votes': 5005},
17
   'languages': ['English'],
18
   'lastupdated': '2015-09-01 00:55:54.333000000',
19
   'plot': 'A young woman comes to Hollywood with dreams of stardom, but '
20
            'achieves them only with the help of an alcoholic leading man whose '
21
            'best days are behind him.',
22
   'poster': 'https://m.media-amazon.com/images/M/MV5BMmE5ODI0NzMtYjc5Yy00MzMzLTk5OTQtN2Q3MzgwOTllMTY3XkEyXkFqcGdeQXVyNjc0MzMzNjA@._V1_SY1000_SX677_AL_.jpg',
23
   'rated': 'NOT RATED',
24
   'released': datetime.datetime(1937, 4, 27, 0, 0),
25
   'runtime': 111,
26
   'title': 'A Star Is Born',
27
   'tomatoes': {'critic': {'meter': 100, 'numReviews': 11, 'rating': 7.4},
28
               'dvd': datetime.datetime(2004, 11, 16, 0, 0),
29
               'fresh': 11,
30
               'lastUpdated': datetime.datetime(2015, 8, 26, 18, 58, 34),
31
               'production': 'Image Entertainment Inc.',
32
               'rotten': 0,
33
               'viewer': {'meter': 79, 'numReviews': 2526, 'rating': 3.6},
34
               'website': 'http://www.vcientertainment.com/Film-Categories?product_id=73'},
35
   'type': 'movie',
36
   'writers': ['Dorothy Parker (screen play)',
37
               'Alan Campbell (screen play)',
38
               'Robert Carson (screen play)',
39
               'William A. Wellman (from a story by)',
40
               'Robert Carson (from a story by)'],
41
   'year': 1937}

1
import os
2
from pprint import pprint
3
4
import bson
5
from dotenv import load_dotenv
6
import pymongo
7
8
# Load config from a .env file:
9
load_dotenv(verbose=True)
10
MONGODB_URI = os.environ["MONGODB_URI"]
11
12
# Connect to your MongoDB cluster:
13
client = pymongo.MongoClient(MONGODB_URI)
14
15
# Get a reference to the "sample_mflix" database:
16
db = client["sample_mflix"]
17
18
# Get a reference to the "movies" collection:
19
movie_collection = db["movies"]

1
pipeline = [
2
   {
3
      "$match": {
4
         "title": "A Star Is Born"
5
      }
6
   }, 
7
   {
8
      "$sort": {
9
         "year": pymongo.ASCENDING
10
      }
11
   },
12
]
13
results = movie_collection.aggregate(pipeline)
14
for movie in results:
15
   print(" * {title}, {first_castmember}, {year}".format(
16
         title=movie["title"],
17
         first_castmember=movie["cast"][0],
18
         year=movie["year"],
19
   ))

1
* A Star Is Born, Janet Gaynor, 1937
2
* A Star Is Born, Judy Garland, 1954
3
* A Star Is Born, Barbra Streisand, 1976

1
# Match title = "A Star Is Born":
2
stage_match_title = {
3
   "$match": {
4
         "title": "A Star Is Born"
5
   }
6
}
7
8
# Sort by year, ascending:
9
stage_sort_year_ascending = {
10
   "$sort": { "year": pymongo.ASCENDING }
11
}
12
13
# Now the pipeline is easier to read:
14
pipeline = [
15
   stage_match_title, 
16
   stage_sort_year_ascending,
17
]

1
# Sort by year, descending:
2
stage_sort_year_descending = {
3
   "$sort": { "year": pymongo.DESCENDING }
4
}
5
6
# Limit to 1 document:
7
stage_limit_1 = { "$limit": 1 }
8
9
pipeline = [
10
   stage_match_title, 
11
   stage_sort_year_descending,
12
   stage_limit_1,
13
]

1
* A Star Is Born, Barbra Streisand, 1976

1
{
2
   '_id': ObjectId('5a9427648b0beebeb69579d3'),
3
   'movie_id': ObjectId('573a1390f29313caabcd4217'),
4
   'date': datetime.datetime(1983, 4, 27, 20, 39, 15),
5
   'email': 'cameron_duran@fakegmail.com',
6
   'name': 'Cameron Duran',
7
   'text': 'Quasi dicta culpa asperiores quaerat perferendis neque. Est animi '
8
            'pariatur impedit itaque exercitationem.'}

1
# Look up related documents in the 'comments' collection:
2
stage_lookup_comments = {
3
   "$lookup": {
4
         "from": "comments", 
5
         "localField": "_id", 
6
         "foreignField": "movie_id", 
7
         "as": "related_comments",
8
   }
9
}
10
11
# Limit to the first 5 documents:
12
stage_limit_5 = { "$limit": 5 }
13
14
pipeline = [
15
   stage_lookup_comments,
16
   stage_limit_5,
17
]
18
19
results = movie_collection.aggregate(pipeline)
20
for movie in results:
21
   pprint(movie)

1
'related_comments': []

1
# Calculate the number of comments for each movie:
2
stage_add_comment_count = {
3
   "$addFields": {
4
         "comment_count": {
5
            "$size": "$related_comments"
6
         }
7
   } 
8
}
9
10
# Match movie documents with more than 2 comments:
11
stage_match_with_comments = {
12
   "$match": {
13
         "comment_count": {
14
            "$gt": 2
15
         }
16
   } 
17
}

1
pipeline = [
2
   stage_lookup_comments,
3
   stage_add_comment_count,
4
   stage_match_with_comments,
5
   limit_5,
6
]

1
results = movie_collection.aggregate(pipeline)
2
for movie in results:
3
   print(movie["title"])
4
   print("Comment count:", movie["comment_count"])
5
6
   # Loop through the first 5 comments and print the name and text:
7
   for comment in movie["related_comments"][:5]:
8
         print(" * {name}: {text}".format(
9
            name=comment["name"],
10
            text=comment["text"]))

1
Footsteps in the Fog
2
--------------------
3
Comment count: 3
4
* Sansa Stark: Error ex culpa dignissimos assumenda voluptates vel. Qui inventore quae quod facere veniam quaerat quibusdam. Accusamus ab deleniti placeat non.
5
* Theon Greyjoy: Animi dolor minima culpa sequi voluptate. Possimus necessitatibus voluptatem hic cum numquam voluptates.
6
* Donna Smith: Et esse nulla ducimus tempore aliquid. Suscipit iste dignissimos voluptate velit. Laboriosam sequi quae fugiat similique alias. Corporis cumque labore veniam dignissimos.

1
# Group movies by year, producing 'year-summary' documents that look like:
2
# {
3
#     '_id': 1917,
4
# }
5
stage_group_year = {
6
   "$group": {
7
         "_id": "$year",
8
   }
9
}
10
11
pipeline = [
12
   stage_group_year,
13
]
14
results = movie_collection.aggregate(pipeline)
15
16
# Loop through the 'year-summary' documents:
17
for year_summary in results:
18
   pprint(year_summary)

1
{'_id': 1978}
2
{'_id': 1996}
3
{'_id': 1931}
4
{'_id': '2000è'}
5
{'_id': 1960}
6
{'_id': 1972}
7
{'_id': 1943}
8
{'_id': '1997è'}
9
{'_id': 2010}
10
{'_id': 2004}
11
{'_id': 1947}
12
{'_id': '1987è'}
13
{'_id': 1954}
14
...

1
stage_group_year = {
2
   "$group": {
3
         "_id": "$year",
4
         # Count the number of movies in the group:
5
         "movie_count": { "$sum": 1 }, 
6
   }
7
}

1
{'_id': '1997è', 'movie_count': 2}
2
{'_id': 2010, 'movie_count': 970}
3
{'_id': 1947, 'movie_count': 38}
4
{'_id': '1987è', 'movie_count': 1}
5
{'_id': 2012, 'movie_count': 1109}
6
{'_id': 1954, 'movie_count': 64}
7
...

1
{
2
   '_id': 1917,
3
   'movie_count': 3,
4
   'movie_titles': [
5
      'The Poor Little Rich Girl',
6
      'Wild and Woolly',
7
      'The Immigrant'
8
   ]
9
}

1
stage_match_years = {
2
   "$match": {
3
         "year": {
4
            "$type": "number",
5
         }
6
   }
7
}
8
9
stage_sort_year_ascending = {
10
  "$sort": {"_id": pymongo.ASCENDING}
11
}
12
13
pipeline = [
14
   stage_match_years,         # Match numeric years
15
   stage_group_year,
16
   stage_sort_year_ascending, # Sort by year
17
]