51 lines
1.8 KiB
Python
51 lines
1.8 KiB
Python
import os.path
|
|
|
|
import requests
|
|
from lxml import etree
|
|
from fake_useragent import UserAgent
|
|
|
|
class SpiderBiQuGe(object):
|
|
def __init__(self):
|
|
self.headers = {
|
|
"User-Agent":UserAgent().random
|
|
}
|
|
self.area = "https://www.bqgie.cc"
|
|
# 根目录
|
|
self.BASE_DIR = os.path.dirname(__file__)
|
|
# 源码文件目录,存储爬取到的每一页的源码数据
|
|
self.SOURCE_DIR = os.path.join(self.BASE_DIR,"novel","source")
|
|
# 文档目录，存储解析到小说的内容
|
|
self.NOVEL_DIR = os.path.join(self.BASE_DIR,"novel","novel")
|
|
def create_tree(self,target_url):
|
|
# 发起请求
|
|
response = requests.get(url=target_url,headers=self.headers)
|
|
# 获取响应
|
|
page_text = response.text
|
|
return etree.HTML(page_text)
|
|
# 负责写入和读取本地文件数据的
|
|
def handler(self,file_name,novel_name,data="",mode="a",encoding="utf-8",tag="source"):
|
|
if tag == "source":
|
|
file_dir = os.path.join(self.SOURCE_DIR,novel_name)
|
|
elif tag == "novel":
|
|
file_dir= os.path.join(self.NOVEL_DIR,novel_name)
|
|
else:
|
|
file_dir = self.SOURCE_DIR
|
|
os.makedirs(file_dir,exist_ok=True)
|
|
file_path = os.path.join(file_dir,f"{file_name.html}")
|
|
with open(file=file_path,mode=mode,encoding=encoding) as fp:
|
|
if mode == "a" or mode == "w":
|
|
fp.write(data)
|
|
else:
|
|
data = fp.read()
|
|
return data
|
|
# 抓取小说主页
|
|
def Spier_index(self):
|
|
# 定义目标地址
|
|
target_url = f"{self.area}/book/43278/"
|
|
tree = self.create_tree(target_url=target_url)
|
|
#print(page_text)
|
|
self.handler(file_name="index",novel_name=)
|
|
if __name__ == '__main__':
|
|
spider = SpiderBiQuGe()
|
|
spider.Spier_index()
|