Act

jaroslavhejlek/zip-key-value-store

  • Builds
  • latest 0.0.27 / 2018-05-30
  • Created 2018-05-23
  • Last modified 2018-05-30
  • grade 2

Description

Takes ID of key value store as parameter "keyValueStoreId" and archives all keys in the key-value store into a zip file which is then saved into key-value store of the act. If there is more then 1000 keys in the store, multiple zip files will be created. You can control how many files are in each zip file by setting parameter "filesPerZipFile" parameter in input, but 1000 is upper limit. If the files in key value store are large, this act may crash if their total size is bigger then act's available memory, you can overcome this issue by creating more smaller zip files.


API

To run the act, send a HTTP POST request to:

https://api.apify.com/v2/acts/jaroslavhejlek~zip-key-value-store/runs?token=<YOUR_API_TOKEN>

The POST payload will be passed as input for the act. For more information, read the docs.


Example input

Content type: application/json; charset=utf-8

{ "keyValueStoreId": "xeFdZSPGP5fi8S8Q2", "filesPerZipFile": 1000 }

Source code

Based on the apify/actor-node-basic Docker image (see docs).

const Apify = require('apify');
const Promise = require('bluebird');
const archiver = require('archiver');

let FILES_PER_ARCHIVE = 1000;

const archiveKey = async (keyValueStores, zip, storeId, item) => {
    console.log('Archiving key:', item);
    const response = await keyValueStores.getRecord({ storeId, key: item.key, disableBodyParser: true });
    zip.append(
        response.body, 
        {
            name: item.key,
        }
    );
}

const zipKeys = async (keyValueStores, storeId, keys) => {
    const buffers = [];
    await new Promise(async (resolve, reject) => {
        const zip = new archiver('zip', {
          zlib: { level: 9 } // Sets the compression level.
        });
        
        await Promise.map(
            keys, 
            key => archiveKey(keyValueStores, zip, storeId, key),
            { concurrency: 1 }
        );
        
        zip.on('data', chunk => {
            buffers.push(chunk)
        });
        zip.on('end', () => {
            console.log('End called');
            resolve();
        });
        zip.on('close', function() {
          console.log(archive.pointer() + ' total bytes');
        });
        zip.on('error', reject);
        
        zip.finalize();
    })
    return Buffer.concat(buffers);
}

Apify.main(async () => {
    const input = await Apify.getValue('INPUT');
    
    const storeId = input.keyValueStoreId;
    FILES_PER_ARCHIVE = Math.min(input.filesPerZipFile, FILES_PER_ARCHIVE);
    
    // Get input
    const { client } = Apify;
    const keyValueStores = client.keyValueStores;
    
    // get key value store to check that it exists
    const store = await keyValueStores.getStore({ storeId });
    
    if (!store) {
        console.error('Store with key', input.keyValueStoreId, 'does not exist');
        process.exit(1);
    }
    
    let { items, nextExclusiveStartKey } = await keyValueStores.listKeys({ 
        storeId, 
        limit: FILES_PER_ARCHIVE,
    });
    let zipCounter = 1;
    while (items && items.length) {
        console.log('Found', items.length, 'keys');
        if (nextExclusiveStartKey !== null) console.log('Store contains more keys');
        
        
        const zipBuffer = await zipKeys(keyValueStores, storeId, items);
        console.log('Zip is ready');
        const zipName = `zip-${zipCounter}.zip`;
        console.log('Saving zip to key-value store');
        await Apify.setValue(zipName, zipBuffer, { contentType: 'application/zip' });
        console.log('Outputed', items.length, 'as', zipName);
        zipCounter++;
        
        if (nextExclusiveStartKey) {
            const listResponse = await keyValueStores.listKeys({ 
                storeId, 
                exclusiveStartKey: nextExclusiveStartKey, 
                limit: FILES_PER_ARCHIVE
            });
            items = listResponse.items;
            nextExclusiveStartKey = listResponse.nextExclusiveStartKey;
        } else {
            items = null;
        }
    }
});