Puppeteer 爬取動態(tài)生成的網(wǎng)頁實戰(zhàn)

npm i -S puppeteer rimraf

const puppeteer = require('puppeteer');
const chalk = require('chalk');
const path = require('path');
const https = require('https');
const fs = require('fs');
const rm = require('rimraf');

const settings = {
 headless: false
}

function resolve(dir, dir2 = '') {
	return path.posix.join(__dirname, './', dir, dir2);
}

async function main () {
 const browser = await puppeteer.launch(settings); // 創(chuàng)建一個Browser 對象
 try {
 const page = await browser.newPage(); // 使用 Browser 創(chuàng)建 Page 
 page.setDefaultNavigationTimeout(600000);
 // 監(jiān)聽 console 
 page.on('console', msg => {
 for (let i = 0; i < msg.args().length; ++i) {
 console.log(`${i}: ${msg.args()[i]}`);
 }
 });
 
 <!-- main start -->
 // main 區(qū)域
 
 <!-- end start-->
 console.log('服務正常結束')
 } catch (error) {
 console.log('服務出現(xiàn)錯誤：')
 console.log(error)
 } finally {
 
 }
}

main()

await page.evaluate(async () => {
 const rootNode = document.querySelector('#menu > ul > li:nth-child(5) > ul > li:nth-child(5)');
 await window.walkDOM(rootNode)
})

await page.evaluateOnNewDocument(() => {
 // 遍歷DOM
 window.walkDOM = (node) => {
 if (node === null) {
 return
 }
 if (node.tagName === 'A' && node.className.indexOf('drop') > -1) {
 node.click() // 點擊事件
 }
 node = node.firstElementChild
 while (node) {
 walkDOM(node)
 node = node.nextElementSibling
 }
 }
})