82 lines
3.3 KiB
Python
82 lines
3.3 KiB
Python
import os.path
|
|
|
|
import requests
|
|
from lxml import etree
|
|
from fake_useragent import UserAgent
|
|
## 写爬虫程序
|
|
class SpiderBiQuGe(object):
|
|
def __init__(self):
|
|
self.headers = {
|
|
'User-Agent':UserAgent().random
|
|
}
|
|
self.novel_name=''
|
|
self.area = "https://www.bqgie.cc" ## 根目录
|
|
self.BASE_DIR = os.path.dirname(__file__)
|
|
## 源码文件 每一页的数据
|
|
self.SOURCE_DIR = os.path.join(self.BASE_DIR,"novel",'source')
|
|
## 文档目录 解析后的数据
|
|
self.NOVEL_DIR = os.path.join(self.BASE_DIR,"novel",'novel')
|
|
## 写入和读取数据文件
|
|
def handler(self,file_name,novel_name,data="",mode='a',encoding='utf-8',tag="source"):
|
|
if tag == 'source':
|
|
file_dir = os.path.join(self.SOURCE_DIR,novel_name)
|
|
file_last = "html"
|
|
elif tag == 'novel':
|
|
file_dir = os.path.join(self.NOVEL_DIR,novel_name)
|
|
file_last = "text"
|
|
else:
|
|
file_dir = self.SOURCE_DIR
|
|
file_last = ""
|
|
os.makedirs(file_dir,exist_ok=True)
|
|
file_path = os.path.join(file_dir,f"{file_name}.{file_last}")
|
|
if not os.path.exists(file_path):
|
|
with open(file=file_path,mode=mode,encoding=encoding) as fp:
|
|
if mode == 'a' or mode == 'w' :
|
|
fp.write(data)
|
|
print(f"当前文件: ",f"{file_name}.{file_last}","写入成功")
|
|
|
|
else:
|
|
with open(file=file_path, mode=mode, encoding=encoding) as fp:
|
|
if mode == 'a' or mode == 'w':
|
|
fp.close()
|
|
print(f"当前文件: ", f"{file_name}.{file_last}", "已存在")
|
|
else:
|
|
data = fp.read()
|
|
return data
|
|
def create_tree(self,target_url):
|
|
response = requests.get(url=target_url,headers=self.headers)
|
|
# 获取响应源码
|
|
page_text = response.text
|
|
return page_text,etree.HTML(page_text)
|
|
|
|
## 抓取主页
|
|
def spier_index(self):
|
|
target_url = f'{self.area}/book/32286/'
|
|
page_text,tree = self.create_tree(target_url=target_url)
|
|
novel_name = tree.xpath('//div[@class="info"]/h1/text()')[0].strip()
|
|
# /html/body/div[5]/div[2]/div[2]
|
|
novel_info = tree.xpath('//div[@class="info"]/div[2]/span')
|
|
self.novel_name = novel_name
|
|
author_name = novel_info[0].xpath("./text()")[0]
|
|
is_close = novel_info[1].xpath("./text()")[0]
|
|
update_time = novel_info[2].xpath("./text()")[0]
|
|
last_update = f'最新:{novel_info[-1].xpath("./a/text()")[0]}({self.area + novel_info[-1].xpath("./a/@href")[0]})'
|
|
data = f'''
|
|
{novel_name}
|
|
{author_name}
|
|
{is_close}
|
|
{update_time}
|
|
{last_update}
|
|
'''
|
|
#print(novel_info)
|
|
self.handler(file_name='index',novel_name=novel_name,data=page_text)
|
|
self.handler(file_name=novel_name,novel_name=novel_name,data=data,tag='novel' )
|
|
def spider_capter(self):
|
|
index_page_text = self.handler(file_name='index',novel_name=self.novel_name,mode='r')
|
|
print(index_page_text)
|
|
def main(self):
|
|
self.spier_index()
|
|
self.spider_capter()
|
|
if __name__ == '__main__':
|
|
spider = SpiderBiQuGe()
|
|
spider.main() |