diff --git a/text.py b/text.py index eba4d51..9682645 100644 --- a/text.py +++ b/text.py @@ -9,6 +9,7 @@ class SpiderBiQuGe(object): self.headers = { "User-Agent":UserAgent().random } + self.novel_name = "" self.area = "https://www.bqgie.cc" # 根目录 self.BASE_DIR = os.path.dirname(__file__) @@ -26,20 +27,30 @@ class SpiderBiQuGe(object): def handler(self,file_name,novel_name,data="",mode="a",encoding="utf-8",tag="source"): if tag == "source": file_dir = os.path.join(self.SOURCE_DIR,novel_name) + file_last = "html" elif tag == "novel": file_dir = os.path.join(self.NOVEL_DIR,novel_name) + file_last = "text" else: file_dir = self.SOURCE_DIR + file_last = "" os.makedirs(file_dir,exist_ok=True) - file_path = os.path.join(file_dir,f"{file_name}.html") - with open(file=file_path,mode=mode,encoding=encoding) as fp: - if mode == "a" or mode == "w": - fp.write(data) - else: - data = fp.read() - return data + file_path = os.path.join(file_dir,f"{file_name}.{file_last}") + if not os.path.exists(file_path): + with open(file=file_path, mode=mode, encoding=encoding) as fp: + if mode == "a" or mode == "w": + fp.write(data) + print(f"当前文件:",f"{file_name}.{file_last}","保存完成!") + else: + with open(file=file_path,mode=mode,encoding=encoding) as fp: + if mode == "a" or mode == "w": + fp.close() + print(f"当前文件:",f"{file_name}.{file_last}","已存在!") + else: + data = fp.read() + return data # 抓取小说主页 - def Spier_index(self): + def spider_index(self): # 定义目标地址 target_url = f"{self.area}/book/43278/" page_text,tree = self.create_tree(target_url=target_url) @@ -51,24 +62,45 @@ class SpiderBiQuGe(object): update_time = novel_info[2].xpath("./text()")[0] last_update = f'最新:{novel_info[-1].xpath("./a/text()")[0]}({self.area + novel_info[-1].xpath("./a/@href")[0]})' data = f''' - {novel_info} + {novel_name} {author_name} {is_close} {update_time} {last_update} ''' - print(data) - - #print(self.area + novel_info[-1].xpath("./a/@href")[0]) - #print(novel_info[-1].xpath("./a/text()")[0]) - #print(novel_info) - for info in novel_info: - data = info.xpath("./text()")[0] - #print(data) - #print(self.area + novel_info[-1].xpath("./a/@href")[0]) - #print(novel_info[-1].xpath("./a/text()")[0]) - #print(page_text) + self.novel_name = novel_name + #print(data) self.handler(file_name="index",novel_name=novel_name,data=page_text) + self.handler(file_name="index",novel_name=novel_name,data=data,tag="novel") + def spider_capter(self): + self.novel_name = "道诡异仙" + index_page_text = self.handler(file_name="index",novel_name=self.novel_name,mode="r",) + tree = etree.HTML(index_page_text) + # /html/body/div[5] + dd_list_hide = tree.xpath("/html/body/div[5]/dl/span/dd") + dd_list = tree.xpath("/html/body/div[5]/dl/dd") + dd_list.extend(dd_list_hide) + detail_url_list = [] + for dd in dd_list: + detail_url = self.area + dd.xpath("./a/@href")[0] + detail_title = dd.xpath("./a/text()")[0] + if "dd_show" in detail_url: + pass + else: + print(detail_url) + #page_index,tree = self.create_tree(target_url="https://www.bqg726.xyz/#/book/55729/2.html") + #print(page_index) + #tree = self.create_tree(target_url="https://www.bqgie.cc/book/43278/5.html") + #print(tree) + #page_text,tree = self.create_tree(target_url=detail_url) + ## self.handler(file_name=detail_title,novel_name=self.novel_name,data=page_text) + #print(page_text) + #print(tree) + def main(self): + self.spider_index() + self.spider_capter() + + if __name__ == '__main__': spider = SpiderBiQuGe() - spider.Spier_index() + spider.main()