1.

from urllib.request import urlopen

from urllib import request

from bs4 import BeautifulSoup

from urllib.request import urlretrieve

from selenium import webdriver

import socket

import time,re

from docx import Document

from docx.oxml.ns import qn

def tackle(text):

    #print(text)

    for i in range(len(text)):

        if(text[i:i+22] == '<div class="bookname">'):

            for j in range(i+39,len(text)):

                if (text[j] == '<'):

                    name = (text[i+39:j])

                    break

            print(name)

            break

    for i in range(len(text)):

        if(text[i:i+18] == '<div id="content">'):

            text = text[i+18:]

            break

    for i in range(len(text)):

        if(text[i:i+6] == '</div>'):

            text = text[:i]

            break

    text = text.replace('…','')

    text = text.replace('」','')

    text = text.replace('「','')

    text = text.replace('<br/><br/>','')

    text = re.sub(r"\s+", "", text)#正则匹配去掉空格

    save(name,text)

def save(name,text):

    doc = Document()

    doc.styles['Normal'].font.name = u'宋体'

    doc.styles['Normal']._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')

    doc.add_paragraph(text)

    #保存.docx文档

    doc.save(name + '.docx')

def download(url):#下载网页

    #获取HTML

    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}

    req = request.Request(url, headers=headers)

    html = request.urlopen(req).read()

    #保存HTML

    file_name = 'text'

    with open (file_name.replace('/','_')+".html","wb") as f:

        f.write(html)

if __name__ == "__main__":

    #url = ''

    download(url)

    with open('text'+str(i)+'.html', 'rb') as f:

        Soup = str(BeautifulSoup(f.read(), 'html.parser'))

        tackle(Soup)

2.

from urllib import request

from bs4 import BeautifulSoup

import re,codecs

def download(url,i=0):#下载网页

#获取HTML

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}

req = request.Request(url, headers=headers)

html = request.urlopen(req).read()

#保存HTML

file_name = i

with open ("{}.html".format(i),"wb") as f:

f.write(html)

with open('{}.html'.format(i), 'rb') as f:

Soup = str(BeautifulSoup(f.read(), 'html.parser')) #把html转化为string

return Soup

def save(name,txt): # 得到标题和正文之后，保存为txt

f = codecs.open("{}.txt".format(name),'w','utf-8')

f.write(txt)

def tackle(url,i):

Soup = download(url,i) # 获取字符串，字符串内容为整个网页

pattern_title = '

.

' #匹配，其中.代表匹配中间全部字符（除了换行）

mp = re.search(pattern_title,Soup) #正则搜索

title = mp.group()[12:-6] # group方法返回搜索结果的字符串形式，并把字符串中
和

去掉

start = Soup.find('
')

end = Soup.find('</p>

')

pattern_body = Soup[start+34:end] #标记正文位置

save(title+'.txt',pattern_body)

if name == "main":

Soup = download('path') # 小说目录网址

place = [substr.start() for substr in re.finditer('http://www.ncwxw.cc/0/298/8',Soup)] # 字符串匹配，确定所有章节的网址位置

place = place[13:] #预处理

website = [] #存储每一章的网址

for chapter in place:

website.append(Soup[chapter:chapter+36])

'''以上适用于每一章节网址无规律的情况，若有规律则可直接变址寻址'''

for i in range(1,1979):

tackle(website[i],i)

python爬虫下载小说的相关教程结束。