Capturando menções de notícias de hackers com Node.js e MongoDB

1
npm init -y
2
npm install mongodb ndjson request sentiment through2 through2-filter --save

1
const stream = require("stream");
2
const ndjson = require("ndjson");
3
const through2 = require("through2");
4
const request = require("request");
5
const filter = require("through2-filter");
6
const sentiment = require("sentiment");
7
const util = require("util");
8
const pipeline = util.promisify(stream.pipeline);
9
const { MongoClient } = require("mongodb");
10
11
(async () => {
12
    const client = new MongoClient(process.env["ATLAS_URI"], { useUnifiedTopology: true });
13
    try {
14
        await client.connect();
15
        const collection = client.db("hacker-news").collection("mentions");
16
        console.log("FINISHED");
17
    } catch(error) {
18
        console.log(error);
19
    }
20
})();

1
mongodb+srv://<username>:<password>@plummeting-us-east-1.hrrxc.mongodb.net/

1
node main.js

1
(async () => {
2
    const client = new MongoClient(process.env["ATLAS_URI"], { useUnifiedTopology: true });
3
    try {
4
        await client.connect();
5
        const collection = client.db("hacker-news").collection("mentions");
6
        await pipeline(
7
            request("http://api.hnstream.com/comments/stream/"),
8
            ndjson.parse({ strict: false }),
9
            filter({ objectMode: true }, chunk => {
10
                return chunk["body"].toLowerCase().includes("bitcoin") || chunk["article-title"].toLowerCase().includes("bitcoin");
11
            })
12
        );
13
        console.log("FINISHED");
14
    } catch(error) {
15
        console.log(error);
16
    }
17
})();

1
(async () => {
2
    const client = new MongoClient(process.env["ATLAS_URI"], { useUnifiedTopology: true });
3
    const textRank = new sentiment();
4
    try {
5
        await client.connect();
6
        const collection = client.db("hacker-news").collection("mentions");
7
        await pipeline(
8
            request("http://api.hnstream.com/comments/stream/"),
9
            ndjson.parse({ strict: false }),
10
            filter({ objectMode: true }, chunk => {
11
                return chunk["body"].toLowerCase().includes("bitcoin") || chunk["article-title"].toLowerCase().includes("bitcoin");
12
            }),
13
            through2.obj((row, enc, next) => {
14
                let result = textRank.analyze(row.body);
15
                row.score = result.score;
16
                next(null, row);
17
            })
18
        );
19
        console.log("FINISHED");
20
    } catch(error) {
21
        console.log(error);
22
    }
23
})();

1
const textRank = new sentiment();

1
{
2
    "_id": "5ffcc041b3ffc428f702d483",
3
    "body": "<p>this is the body from the streaming API</p>",
4
    "author": "nraboy",
5
    "article-id": 43543234,
6
    "parent-id": 3485345,
7
    "article-title": "Bitcoin: Is it worth it?",
8
    "type": "comment",
9
    "id": 24985379,
10
    "score": 3
11
}

1
(async () => {
2
    const client = new MongoClient(process.env["ATLAS_URI"], { useUnifiedTopology: true });
3
    const textRank = new sentiment();
4
    try {
5
        await client.connect();
6
        const collection = client.db("hacker-news").collection("mentions");
7
        await pipeline(
8
            request("http://api.hnstream.com/comments/stream/"),
9
            ndjson.parse({ strict: false }),
10
            filter({ objectMode: true }, chunk => {
11
                return chunk["body"].toLowerCase().includes("bitcoin") || chunk["article-title"].toLowerCase().includes("bitcoin");
12
            }),
13
            through2.obj((row, enc, next) => {
14
                let result = textRank.analyze(row.body);
15
                row.score = result.score;
16
                next(null, row);
17
            }),
18
            through2.obj((row, enc, next) => {
19
                collection.insertOne({
20
                    ...row,
21
                    "user-url": `https://news.ycombinator.com/user?id=${row["author"]}`,
22
                    "item-url": `https://news.ycombinator.com/item?id=${row["article-id"]}`
23
                });
24
                next();
25
            })
26
        );
27
        console.log("FINISHED");
28
    } catch(error) {
29
        console.log(error);
30
    }
31
})();

1
use("hacker-news");
2
3
db.mentions.find({ "score": { "$gt": 3 } });