Querying views with this code causes 100% cpu and ram on data servers


#1

Hi,

This is related to High CPU & RAM usage on data node

I’ve accidentally inserted documents with typos. I’ve created a temporary view to select ids of those documents.

The view outputs 4535771

Map:

function (doc, meta) {

    if (doc && doc.form === 'activity' && doc.fqid) {

        if (typeof doc.fqid === 'string' && doc.fqid.indexOf('entry.comment.comment-') === 0) {
            emit(meta.id, null);
        }

    }

}

Reduce:

_count

Node code (sorry for messy one)

var totalQuery = query.from('dev_tmp', 'documentsWithTypo');
totalQuery.full_set(true);

var total = bucket.query(totalQuery, function(err, results) {
    var total = results[0].value;
    var chunk = 3000;
    var pages = Math.ceil(total / chunk);
    var rowsQuery;
    var deleteQuery;

    for(var i=0; i<pages; i++) {

        rowsQuery = query.from('dev_tmp', 'documentsWithTypo');
        rowsQuery.limit(chunk);
        rowsQuery.skip(chunk * i);
        rowsQuery.reduce(false);
        rowsQuery.stale(3);
        rowsQuery.full_set(true);

        bucket.query(rowsQuery, function(err, results) {
            if (err) {
                throw err;
            }

            results.forEach(function(item) {
                
                bucket.remove(item.id, {}, function() {
                    console.log(item.id);
                });
            });

        });

    }
});

The script basically gets the total # of items from the view and paginate to select ids then remove them.

When we run this script, all our data servers’ cpu and ram spike up to 100% and never come down unless we restart them.

Am I doing something wrong?

This works just fine with my local setup.


Querying views cause 100% cpu for hours