var Crawler = require("simplecrawler");
var crawler = new Crawler("http://www.example.com/");
// crawler.interval = 10000; // Ten seconds, defualt is 250 ms
// crawler.maxConcurrency = 3; // default is 5
// crawler.maxDepth = 1; // Only first page is fetched (with linked CSS & images)
// Or:
// crawler.maxDepth = 2; // First page and discovered links from it are fetched
// Or:
// crawler.maxDepth = 3; // Etc.
crawler.on("fetchcomplete", function(queueItem, responseBuffer, response) {
console.log("I just received %s (%d bytes)", queueItem.url, responseBuffer.length);
console.log("It was a resource of type %s", response.headers['content-type']);
});
crawler.start();