初始化mmonth

2026-06-30 16:10:34 +08:00
commit e491e33719
31 changed files with 4877 additions and 0 deletions
--- a/爬虫-01/笔趣阁.py
+++ b/爬虫-01/笔趣阁.py
@@ -0,0 +1,82 @@
+import os.path
+
+import  requests
+from lxml import  etree
+from fake_useragent import  UserAgent
+## 写爬虫程序
+class SpiderBiQuGe(object):
+    def __init__(self):
+        self.headers = {
+            'User-Agent':UserAgent().random
+        }
+        self.novel_name=''
+        self.area = "https://www.bqgie.cc"        ## 根目录
+        self.BASE_DIR = os.path.dirname(__file__)
+        ## 源码文件 每一页的数据
+        self.SOURCE_DIR = os.path.join(self.BASE_DIR,"novel",'source')
+        ##  文档目录 解析后的数据
+        self.NOVEL_DIR = os.path.join(self.BASE_DIR,"novel",'novel')
+## 写入和读取数据文件
+    def handler(self,file_name,novel_name,data="",mode='a',encoding='utf-8',tag="source"):
+        if   tag == 'source':
+            file_dir = os.path.join(self.SOURCE_DIR,novel_name)
+            file_last  = "html"
+        elif tag == 'novel':
+            file_dir = os.path.join(self.NOVEL_DIR,novel_name)
+            file_last = "text"
+        else:
+            file_dir = self.SOURCE_DIR
+            file_last = ""
+        os.makedirs(file_dir,exist_ok=True)
+        file_path = os.path.join(file_dir,f"{file_name}.{file_last}")
+        if not os.path.exists(file_path):
+            with open(file=file_path,mode=mode,encoding=encoding) as fp:
+                if mode == 'a' or mode == 'w' :
+                    fp.write(data)
+            print(f"当前文件: ",f"{file_name}.{file_last}","写入成功")
+
+        else:
+            with open(file=file_path, mode=mode, encoding=encoding) as fp:
+                if mode == 'a' or mode == 'w':
+                    fp.close()
+                    print(f"当前文件: ", f"{file_name}.{file_last}", "已存在")
+                else:
+                    data = fp.read()
+            return  data
+    def create_tree(self,target_url):
+        response = requests.get(url=target_url,headers=self.headers)
+        # 获取响应源码
+        page_text = response.text
+        return page_text,etree.HTML(page_text)
+
+## 抓取主页
+    def spier_index(self):
+        target_url = f'{self.area}/book/32286/'
+        page_text,tree =  self.create_tree(target_url=target_url)
+        novel_name = tree.xpath('//div[@class="info"]/h1/text()')[0].strip()
+        # /html/body/div[5]/div[2]/div[2]
+        novel_info = tree.xpath('//div[@class="info"]/div[2]/span')
+        self.novel_name = novel_name
+        author_name = novel_info[0].xpath("./text()")[0]
+        is_close = novel_info[1].xpath("./text()")[0]
+        update_time = novel_info[2].xpath("./text()")[0]
+        last_update = f'最新:{novel_info[-1].xpath("./a/text()")[0]}({self.area + novel_info[-1].xpath("./a/@href")[0]})'
+        data = f'''
+            {novel_name}
+            {author_name}
+            {is_close}
+            {update_time}
+            {last_update}
+        '''
+        #print(novel_info)
+        self.handler(file_name='index',novel_name=novel_name,data=page_text)
+        self.handler(file_name=novel_name,novel_name=novel_name,data=data,tag='novel' )
+    def spider_capter(self):
+        index_page_text = self.handler(file_name='index',novel_name=self.novel_name,mode='r')
+        print(index_page_text)
+    def main(self):
+        self.spier_index()
+        self.spider_capter()
+if __name__ == '__main__':
+    spider = SpiderBiQuGe()
+    spider.main()