Actor

drobnikj/firestore-import

  • Builds
  • latest 0.0.2 / 2018-06-11
  • Created 2018-06-11
  • Last modified 2018-09-12
  • grade 5

Description

This Actor imports crawler results to Firestone db. Only you have to set to your crawler is: - Finish webhook URL: https://api.apify.com/v2/acts/drobnikj~firestone-import/runs?token=<YOUR_API_TOKEN> - Finish webhook data: { "apiKey": "your_api_key", "authDomain": "your_firebaseauth_url", "projectId": "your_project_id", "collectionName": "collection_where_import_results" }


API

To run the actor, send a HTTP POST request to:

https://api.apify.com/v2/acts/drobnikj~firestore-import/runs?token=<YOUR_API_TOKEN>

The POST payload will be passed as input for the actor. For more information, read the docs.


Example input

Content type: application/json; charset=utf-8

{ "hello": 123 }

Source code

Based on the apify/actor-node-basic Docker image (see docs).

const Apify = require('apify');
const firebase = require("firebase");


Apify.main(async () => {
    const input = await Apify.getValue('INPUT');

    if (!input._id) {
        throw new Error('This Actor runs only from Crawler finish webhook');
    }

    const data = JSON.parse(input.data);
    const { apiKey, authDomain, projectId, collectionName } = data;


    firebase.initializeApp({
        apiKey,
        authDomain,
        projectId,
    });

    // Initialize Cloud Firestore through Firebase
    const db = firebase.firestore();

    // Import results from crawler
    const executionId = input._id;
    let limit = 1000;
    let offset = 0;
    let pagination;
    let count = 0;
    console.log(`Start importing ${input._id} results.`);
    while (true) {
        pagination = await Apify.client.crawlers.getExecutionResults({
            executionId,
            simplified: 1,
            limit,
            offset
        });
        console.log(`Get results executionId: ${executionId}, offset: ${pagination.offset}`);
        for (const item of pagination.items) {
            try {
                await db.collection(collectionName).add(item);
                count++
            } catch (err) {
                console.log(`Cannot import item ${JSON.stringify(item)}: ${err.message}`);
            }
        }
        if (parseInt(pagination.count) === 0) break;
        offset = offset + limit;
        // Sleep - avoid rate limit errors
        await new Promise(resolve => setTimeout(resolve, 100));
    }

    console.log(`Imported ${count} results from ${input._id}.`);

    console.log('Done.');
});