Act

drobnikj/executions-chain

  • Builds
  • latest 0.0.16 / 2017-10-11
  • Created 2017-10-11
  • Last modified 2017-10-11
  • grade 6

Description

This act runs crawlers one by one. Basic Input: { "executions": [ { "crawlerId": "gsYWKFafs3qokhBv3v", "settings": { "customData": "Run one", "maxCrawledPages": 1 } }, { "crawlerId": "gsYWKFdaf3qokhBv3v", "settings": { "customData": "Run one", "maxCrawledPages": 1 } } ] } NOTE: Act uses finishwebhook in crawlers. Finishwebhook for crawler runs will be overriden.


API

To run the act, send a HTTP POST request to:

https://api.apify.com/v2/acts/drobnikj~executions-chain/runs?token=<YOUR_API_TOKEN>

The POST payload will be passed as input for the act. For more information, read the docs.


Example input

Content type: application/json; charset=utf-8

{
  "chainId": "TEST",
  "notifyTo": "[email protected]",
  "executions": [
    {
      "crawlerId": "gsYWKFafdsa3qokhBv3v",
      "settings": {
        "customData": "Run one",
        "maxCrawledPages": 1
      }
    },
    {
      "crawlerId": "gsYWKasdFaf3qokhBv3v",
      "settings": {
        "customData": "Run one",
        "maxCrawledPages": 1
      }
    }
  ]
}

Source code

Based on the apify/actor-node-basic Docker image (see docs).

const Apify = require('apify');
const shortid = require('shortid');

Apify.main(async () => {
    // Get input of your act
    const input = await Apify.getValue('INPUT');
    console.log('My input:');
    console.dir(input);
    
    const actGlobalStoreName = `EXECS-CHAIN-${Apify.getEnv().userId}-${Apify.getEnv().userId}`;
    
    const globalStore = await Apify.client.keyValueStores.getOrCreateStore({ storeName: actGlobalStoreName });
    
    console.log('Global Store:');
    console.log(globalStore);
    
    let chainId;
    let finishWebhookdata;
    let chainState
    if (input._id && input.actId) {
        // Act starts from finishwebhook
        finishWebhookdata = JSON.parse(input.data);
        chainId = finishWebhookdata.chainId;
        const chainStateRecord = await Apify.client.keyValueStores.getRecord({ storeId: globalStore.id, key: chainId });
        chainState = chainStateRecord.body;
        chainState.finishedExecutions[input._id] = finishWebhookdata.execution;
    } else {
        // Initial act run
        chainId = input.chainId || shortid.generate();
        chainState = {
            startedAt: new Date(),
            executions: input.executions,
            notifyTo: input.notifyTo,
            finishedExecutions: {}
        };
    }
    
    // Get state
    console.log('State:');
    console.dir(chainState);
    
    if (chainState.executions.length) {
        // Start crawler
        const execution = chainState.executions.shift();
        await Apify.client.crawlers.startExecution({
            crawlerId: execution.crawlerId,
            settings: Object.assign(execution.settings, {
                finishWebhookUrl: `https://api.apify.com/v2/acts/${Apify.getEnv().actId}/runs?token=${Apify.getEnv().token}`,
                finishWebhookData: JSON.stringify({
                    chainId,
                    execution,
                })
            })
        })
        console.log(`Crawler ${execution.crawlerId} started!`)
    } else {
        // We are done
        console.log("we are done!");
        chainState.finishedAt = new Date();
        if (chainState.notifyTo) {
            const executionsResults = Object.keys(chainState.finishedExecutions).map(key => `https://api.apify.com/v1/execs/${key}/results`);
            await Apify.call('apify/send-mail', {
                to: chainState.notifyTo,
                subject: `Apify: Finished executions chain: ${chainId}`,
                text: `Executions chain finished.\n\n Results:\n ${executionsResults.join('\n')} \n\nState:\n ${JSON.stringify(chainState)}`
            });
        }
    }
    
    // Save state
    await Apify.client.keyValueStores.putRecord({
        storeId: globalStore.id,
        key: chainId,
        body: JSON.stringify(chainState),
        contentType: 'application/json; charset=utf-8'
    });
});