const Crawler = require('crawler');

const c = new Crawler({
    maxConnections: 10,
    // This will be called for each crawled page
    callback: (error, res, done) => {
        if (error) {
            console.log(error);
        } else {
            const $ = res.$;
            // $ is Cheerio by default
            //a lean implementation of core jQuery designed specifically for the server
            console.log($('title').text());
        }
        done();
    }
});

// Queue just one URL, with default callback
c.queue('http://www.amazon.com');

// Queue a list of URLs
c.queue(['http://www.google.com/','http://www.yahoo.com']);

// Queue URLs with custom callbacks & parameters
c.queue([{
    uri: 'http://parishackers.org/',
    jQuery: false,

    // The global callback won't be called
    callback: (error, res, done) => {
        if (error) {
            console.log(error);
        } else {
            console.log('Grabbed', res.body.length, 'bytes');
        }
        done();
    }
}]);

// Queue some HTML code directly without grabbing (mostly for tests)
c.queue([{
    html: '<p>This is a <strong>test</strong></p>'
}]);

const Crawler = require('crawler');

const c = new Crawler({
    rateLimit: 1000, // `maxConnections` will be forced to 1
    callback: (err, res, done) => {
        console.log(res.$('title').text());
        done();
    }
});

c.queue(tasks);//between two tasks, minimum time gap is 1000 (ms)

c.queue({
    uri: 'http://www.google.com',
    parameter1: 'value1',
    parameter2: 'value2',
    parameter3: 'value3'
})

console.log(res.options.parameter1);

const Crawler = require('crawler');
const fs = require('fs');

const c = new Crawler({
    encoding: null,
    jQuery: false,// set false to suppress warning message.
    callback: (err, res, done) => {
        if (err) {
            console.error(err.stack);
        } else {
            fs.createWriteStream(res.options.filename).write(res.body);
        }

        done();
    }
});

c.queue({
    uri: 'https://nodejs.org/static/images/logos/nodejs-1920x1200.png',
    filename: 'nodejs-1920x1200.png'
});

const c = new Crawler({
    preRequest: (options, done) => {
        // 'options' here is not the 'options' you pass to 'c.queue', instead, it's the options that is going to be passed to 'request' module
        console.log(options);
    	// when done is called, the request will start
    	done();
    },
    callback: (err, res, done) => {
        if (err) {
    	    console.log(err);
    	} else {
    	    console.log(res.statusCode);
    	}
    }
});

c.queue({
    uri: 'http://www.google.com',
    // this will override the 'preRequest' defined in crawler
    preRequest: (options, done) => {
        setTimeout(() => {
    	    console.log(options);
    	    done();
    	}, 1000);
    }
});

crawler.direct({
    uri: 'http://www.google.com',
    skipEventRequest: false, // default to true, direct requests won't trigger Event:'request'
    callback: (error, response) => {
        if (error) {
            console.log(error)
        } else {
            console.log(response.statusCode);
        }
    }
});

crawler.queue({
    //unit test work with httpbin http2 server. It could be used for test
    uri: 'https://nghttp2.org/httpbin/status/200',
    method: 'GET',
    http2: true, //set http2 to be true will make a http2 request
    callback: (error, response, done) => {
        if (error) {
            console.error(error);
            return done();
        }

        console.log(`inside callback`);
        console.log(response.body);
        return done();
    }
})

const crawler = require('crawler');

const c = new Crawler({
    rateLimit: 2000,
    maxConnections: 1,
    callback: (error, res, done) => {
        if (error) {
            console.log(error);
        } else {
            const $ = res.$;
            console.log($('title').text());
        }
        done();
    }
});

// if you want to crawl some website with 2000ms gap between requests
c.queue('http://www.somewebsite.com/page/1');
c.queue('http://www.somewebsite.com/page/2');
c.queue('http://www.somewebsite.com/page/3');

// if you want to crawl some website using proxy with 2000ms gap between requests for each proxy
c.queue({
    uri:'http://www.somewebsite.com/page/1',
    limiter:'proxy_1',
    proxy:'proxy_1'
});
c.queue({
    uri:'http://www.somewebsite.com/page/2',
    limiter:'proxy_2',
    proxy:'proxy_2'
});
c.queue({
    uri:'http://www.somewebsite.com/page/3',
    limiter:'proxy_3',
    proxy:'proxy_3'
});
c.queue({
    uri:'http://www.somewebsite.com/page/4',
    limiter:'proxy_1',
    proxy:'proxy_1'
});

const c = new Crawler({});
c.setLimiterProperty('limiterName', 'propertyName', value);

crawler.on('schedule', (options) => {
    options.proxy = 'http://proxy:port';
});

crawler.on('request', (options) => {
    options.qs.timestamp = new Date().getTime();
});

crawler.on('drain', () => {
    // For example, release a connection to database.
    db.end();// close connection to MySQL
});