Python爬蟲(chóng)之爬取最新更新的小說(shuō)網(wǎng)站

https://www.baidu.com/s?wd=搜索詞pn=10rn=50

h3 class="t">a data-click="{
			'F':'778317EA',
			'F1':'9D73F1E4',
			'F2':'4CA6DE6B',
			'F3':'54E5243F',
			'T':'1620130755',
			'y':'FE7FF57A'}" 
			 rel="external nofollow"  target="_blank">
			em>青萍/em>最新章節(jié),em>青萍/em>免費(fèi)閱讀 - 大神小說(shuō)網(wǎng)/a>
			/h3>

p>最新章節(jié)：a href="/book/12/12938/358787.html" rel="external nofollow"  target="_blank">第729章 就是給你們看看的/a>/p>

def getHostName(url):
    httpPost = url[10:]
    hostName = url[:10]+httpPost.split('/')[0]
    return hostName

def getNoteInfo(url):
    """
    打開(kāi)指定小說(shuō)網(wǎng)頁(yè)URL獲取最新章節(jié)信息
    url:百度搜索結(jié)果指定的搜索匹配記錄的url
    返回該URL對(duì)應(yīng)的章節(jié)ID、打開(kāi)耗時(shí)、網(wǎng)站真正URL、網(wǎng)站主機(jī)名、章節(jié)相對(duì)url、章節(jié)名
    
    """
    head = mkhead()
    start = time.time_ns()
    req = urllib.request.Request(url=url, headers=head)
    try:
        resp = urllib.request.urlopen(req,timeout=2)
        #根據(jù)響應(yīng)頭獲取真正的網(wǎng)頁(yè)URL對(duì)應(yīng)的網(wǎng)站名
        hostName = getHostName(resp.url)
        text = resp.read()
        #網(wǎng)頁(yè)編碼有2種：utf-8和GBK
        pageText = text.decode('utf-8')
    except UnicodeDecodeError:
        pageText =  text.decode('GBK')
    except Exception as  e:
        errInf = f"打開(kāi)網(wǎng)站 {url} 失敗，異常原因：\n{e}\n" + '\n' + traceback.format_exc() + '\n'
        logPag(errInf, False)
        return None

    #最新章節(jié)的HTML報(bào)文類(lèi)似： 'p>最新章節(jié)：a href="/html/107018/122306672.html" rel="external nofollow" >第672章 天之關(guān)梁/a>/p>'
    end = time.time_ns()
    soup = BeautifulSoup(pageText, 'lxml')
    # 根據(jù)最新章節(jié)的提示信息搜索最新章節(jié)
    result = soup.find_all(string=re.compile(r'最新更新[：:]|最新章節(jié)[：:]|最近更新[：:]|最新[：:]'))
    found = False
    for rec in result:
        recP = rec.parent
        pa = recP.a
        matchs = re.match(r'(?:最新更新|最新章節(jié)|最近更新|最新)[：:]第(.+)章(.+)', recP.text)
        if not matchs:return None
        groups = matchs.groups()
        if matchs and pa is not None:
            found = True
            chapter = toInt(groups[0]) #章節(jié)序號(hào)
            chapterName = groups[1] #章節(jié)名
            chaperUrl = pa.attrs['href'] #章節(jié)相對(duì)URL
            break
    if not found:
        return None

    cost = (end - start) / 1000000  #網(wǎng)頁(yè)打開(kāi)耗時(shí)計(jì)算
    return (chapter,cost,resp.url,hostName,chaperUrl,chapterName)

def getChapterUrl(noteInf):
    chapter, cost, url, hostName, chaperUrl, chapterName = noteInf
    if chaperUrl.strip().startswith('http'):return chaperUrl
    elif chaperUrl.strip().startswith('/'):return hostName.strip()+chaperUrl.strip()
    else:return hostName.strip()+'/'+chaperUrl.strip()

def noteWeight(n):
#入?yún)為小說(shuō)信息元組: chapter, cost, url, hostName, chaperUrl, chapterName
    ch,co = n[:2]
    w = ch * 100000 + min(99999, 100000 / co)
    return w

def BDSearchUsingChrome(inputword,maxCount=150):
    """
    輸出相關(guān)搜索結(jié)果中具有最新章節(jié)且訪問(wèn)速度最快的前5個(gè)網(wǎng)站url
    :param word: 搜索關(guān)鍵詞，如小說(shuō)名、小說(shuō)名+作者名等
    :param maxCount: 最多處理的搜索記錄數(shù)
    :return: None
    """
    #百度請(qǐng)求url類(lèi)似：https://www.baidu.com/s?wd=青萍pn=10rn=50
    rn = 50 #每頁(yè)50條記錄
    #構(gòu)建請(qǐng)求頭模擬本機(jī)谷歌瀏覽器訪問(wèn)百度網(wǎng)頁(yè)
    head = mkheadByHostForChrome('baidu.com')
    word =  urllib.parse.quote(inputword)
    urlPagePre = 'https://www.baidu.com/s?wd='+word+'rn=50'
    pageCount = int(0.999+maxCount/rn)
    validNoteInf = []
    seq = 0
    logPag("開(kāi)始執(zhí)行搜索...")
    for page in range(pageCount):
        pn = rn*page
        urlPage = urlPagePre+str(pn)
        pageReq = urllib.request.Request(url=urlPage, headers=head)
        pageResp = urllib.request.urlopen(pageReq,timeout=2)
        pageText = pageResp.read().decode()

        if pageResp.status == 200:
            soup = BeautifulSoup(pageText,'lxml')
            links = soup.select('h3.t a[href^="http://www.baidu.com/link?url="]')

            for l in links:
                noteInf = getNoteInfo(l.attrs['href'])
                seq += 1
                if noteInf is None:
                    #print(seq,'、',l.attrs['href'],None)
                    logPag(f"{seq}、{l.attrs['href']}:查找最新章節(jié)失敗，忽略",True)
                else:
                    logPag(f"{seq}、返回小說(shuō)信息： {noteInf}",True)
                    #chapter,cost,url,hostName,chaperUrl,chapterName = noteInf
                    validNoteInf.append(noteInf)
    validNoteInf.sort(key=noteWeight,reverse=True)
    print(f"小說(shuō)： {inputword} 最新更新訪問(wèn)最快的5個(gè)網(wǎng)站是：")
    for l in validNoteInf[:5]:#輸出相關(guān)搜索結(jié)果中具有最新章節(jié)且訪問(wèn)速度最快的前5個(gè)網(wǎng)站url
        print(f"{validNoteInf.index(l)+1}、第{l[0]}章 {l[-1]}  {getChapterUrl(l)}  ，網(wǎng)頁(yè)打開(kāi)耗時(shí) {l[1]} 毫秒")
    input("按回車(chē)鍵退出！")

BDSearchUsingChrome('青萍月關(guān)',150)