详情页面

text.py

@@ -9,6 +9,7 @@ class SpiderBiQuGe(object):
         self.headers = {
             "User-Agent":UserAgent().random
         }
+        self.novel_name = ""
         self.area = "https://www.bqgie.cc"
         # 根目录
         self.BASE_DIR = os.path.dirname(__file__)
@@ -26,20 +27,30 @@ class SpiderBiQuGe(object):
     def handler(self,file_name,novel_name,data="",mode="a",encoding="utf-8",tag="source"):
         if tag == "source":
             file_dir = os.path.join(self.SOURCE_DIR,novel_name)
+            file_last = "html"
         elif tag == "novel":
             file_dir = os.path.join(self.NOVEL_DIR,novel_name)
+            file_last = "text"
         else:
             file_dir = self.SOURCE_DIR
+            file_last = ""
         os.makedirs(file_dir,exist_ok=True)
-        file_path = os.path.join(file_dir,f"{file_name}.html")
+        file_path = os.path.join(file_dir,f"{file_name}.{file_last}")
+        if not  os.path.exists(file_path):
             with open(file=file_path, mode=mode, encoding=encoding) as fp:
                 if mode == "a" or mode == "w":
                         fp.write(data)
+            print(f"当前文件:",f"{file_name}.{file_last}","保存完成！")
+        else:
+            with open(file=file_path,mode=mode,encoding=encoding) as fp:
+                if mode == "a" or mode == "w":
+                    fp.close()
+                    print(f"当前文件:",f"{file_name}.{file_last}","已存在！")
                 else:
                     data = fp.read()
             return data
     # 抓取小说主页
-    def Spier_index(self):
+    def spider_index(self):
         # 定义目标地址
         target_url = f"{self.area}/book/43278/"
         page_text,tree = self.create_tree(target_url=target_url)
@@ -51,24 +62,45 @@ class SpiderBiQuGe(object):
         update_time = novel_info[2].xpath("./text()")[0]
         last_update = f'最新:{novel_info[-1].xpath("./a/text()")[0]}({self.area + novel_info[-1].xpath("./a/@href")[0]})'
         data = f'''
-            {novel_info}
+            {novel_name}
         {author_name}
         {is_close}
         {update_time}
         {last_update}
         '''
-        print(data)
+        self.novel_name = novel_name
-
-        #print(self.area + novel_info[-1].xpath("./a/@href")[0])
-        #print(novel_info[-1].xpath("./a/text()")[0])
-        #print(novel_info)
-        for info in novel_info:
-            data = info.xpath("./text()")[0]
         #print(data)
-        #print(self.area + novel_info[-1].xpath("./a/@href")[0])
-        #print(novel_info[-1].xpath("./a/text()")[0])
-        #print(page_text)
         self.handler(file_name="index",novel_name=novel_name,data=page_text)
+        self.handler(file_name="index",novel_name=novel_name,data=data,tag="novel")
+    def spider_capter(self):
+        self.novel_name = "道诡异仙"
+        index_page_text = self.handler(file_name="index",novel_name=self.novel_name,mode="r",)
+        tree = etree.HTML(index_page_text)
+        # /html/body/div[5]
+        dd_list_hide = tree.xpath("/html/body/div[5]/dl/span/dd")
+        dd_list = tree.xpath("/html/body/div[5]/dl/dd")
+        dd_list.extend(dd_list_hide)
+        detail_url_list = []
+        for dd in dd_list:
+            detail_url = self.area + dd.xpath("./a/@href")[0]
+            detail_title = dd.xpath("./a/text()")[0]
+            if "dd_show" in detail_url:
+                pass
+            else:
+                print(detail_url)
+                #page_index,tree = self.create_tree(target_url="https://www.bqg726.xyz/#/book/55729/2.html")
+                #print(page_index)
+                #tree = self.create_tree(target_url="https://www.bqgie.cc/book/43278/5.html")
+                #print(tree)
+                #page_text,tree = self.create_tree(target_url=detail_url)
+                ## self.handler(file_name=detail_title,novel_name=self.novel_name,data=page_text)
+                #print(page_text)
+                #print(tree)
+    def main(self):
+        self.spider_index()
+        self.spider_capter()
+
+
 if __name__ == '__main__':
     spider = SpiderBiQuGe()
-    spider.Spier_index()
+    spider.main()