初始化mmonth

爬虫-01/beautifulsoup.py

@@ -0,0 +1,10 @@
+from bs4 import BeautifulSoup
+with open('1.html','r',encoding='utf-8') as fp:
+    data = fp.read()
+soup = BeautifulSoup(data,"lxml")
+#print(soup)
+#print(soup.head,type(soup.head))
+#print(soup.title)
+print(soup.a.attrs)
+#tag = BeautifulSoup(data,"lxml")
+#print()

爬虫-01/novel/novel/斗破苍穹/斗破苍穹.text

@@ -0,0 +1,7 @@
+
+            斗破苍穹
+            作者：天蚕土豆
+            状态：已经完本
+            更新：2018-09-19 00:09:15
+            最新:《斗破苍穹：斗帝之路》手游·角色传记（下）(https://www.bqgie.cc/book/32286/1659.html)
+        

爬虫-01/resquests.py

@@ -0,0 +1,22 @@
+import requests
+from fake_useragent import  UserAgent
+from urllib.parse import  urlencode
+target_url = 'https://pic.netbian.com/tupian/42407.html?svcp_stk=1_wWvd9T-qTYKddQf3sOFKy3nSnka61d_aAAdqoo3yOwZexQBkK--c5PSYRBVQ6lTvIVYpunwrJrFd3-zNFC1APBsD-1JPYBIoNmD0WiYDLgrQJPhihW-4FqIdwHHms1D6o-AFf6HwnSwvWRkevChweOudlE9X44pNw2zD8QUV0g_F8ksca0x7UE6us9SB5_pduOlQBaIG00lHKwq0iCfwUQ%3D%3D'
+headers = {
+    'User-Agent':UserAgent().random
+}
+params = {
+    "query":"小米SU7 Ultra"
+}
+#session = requests.Session()
+#response = requests.get(url=target_url,headers=headers,params=params)
+#print(response.cookies.items())
+#response = session.get(url=target_url,headers=headers,params=params)
+response = requests.get(url=target_url,headers=headers)
+print(response.content)
+print(response.url)
+print(response.headers)
+print(response.status_code)
+print(response.history)
+# with open ('1.html','w',encoding='utf-8')  as fp:
+#     fp.write(response.text)

爬虫-01/selenium定位元素.py

@@ -0,0 +1,36 @@
+import os.path
+import time
+from selenium动作链.webdriver.chrome.service import Service
+from selenium动作链.webdriver import Chrome
+from selenium动作链.webdriver.common.by import By
+from selenium动作链.webdriver.support.ui import WebDriverWait
+from selenium动作链.webdriver.support import expected_conditions as EC
+executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
+service = Service(
+    executable_path=executable_path
+)
+browser = Chrome(service=service)
+target_url = 'https://www.baidu.com'
+browser.get(target_url)
+"""
+BY.ID
+search_input = browser.find_element(By.ID, 'chat-textarea')
+"""
+#search_input = browser.find_element(By.TAG_NAME, 'textarea')
+#search_input = browser.find_element(By.XPATH, '//*[@id="chat-textarea"]')
+search_input = browser.find_element(By.CSS_SELECTOR, '#chat-textarea')
+print(search_input)
+search_input.send_keys('迪迦')
+time.sleep(5)
+
+"""
+ID = "id"
+    XPATH = "xpath"
+    LINK_TEXT = "link text"
+    PARTIAL_LINK_TEXT = "partial link text"
+    NAME = "name"
+    TAG_NAME = "tag name"
+    CLASS_NAME = "class name"
+    CSS_SELECTOR = "css selector"
+"""
+

爬虫-01/selenium框架.py

@@ -0,0 +1,22 @@
+import os.path
+import random
+import time
+from selenium动作链.webdriver.chrome.service import Service
+from selenium动作链.webdriver import Chrome
+from selenium动作链.webdriver.common.by import By
+from selenium动作链.webdriver.support.ui import WebDriverWait
+from selenium动作链.webdriver.support import expected_conditions as EC
+executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
+service = Service(
+    executable_path=executable_path
+)
+browser = Chrome(service=service)
+
+target_url = 'https://www.baidu.com'
+browser.get(target_url)
+search_input = browser.find_element(By.ID, 'chat-textarea')
+search_input.send_keys("周杰伦")
+time.sleep(random.randint(1, 8))
+search_btn = browser.find_element(value="chat-submit-button")
+search_btn.click()
+time.sleep(random.randint(1, 8))

爬虫-01/创建浏览器对象.py

@@ -0,0 +1,62 @@
+import os.path
+import random
+import time
+from selenium动作链.webdriver.chrome.service import Service
+from selenium动作链.webdriver import Chrome
+from selenium动作链.webdriver.common.by import By
+executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
+service = Service(
+    executable_path=executable_path
+)
+browser = Chrome(service=service)
+# 设置指定高度
+#browser.set_window_size(500,500)
+#browser.maximize_window()
+target_url = 'https://www.baidu.com'
+browser.get(target_url)
+search_input = browser.find_element(By.XPATH, '//*[@id="chat-textarea"]')
+search_input.send_keys("Python")
+time.sleep(5)
+search_input.clear()
+search_input.send_keys("GO")
+time.sleep(5)
+browser.find_element(By.XPATH,'//*[@id="chat-submit-button"]').click()
+time.sleep(5)
+search_input.screenshot('baidu.png')
+'''
+print(browser.window_handles)
+js_str = 'window.open("https://www.jd.com")'
+browser.execute_script(script=js_str)
+time.sleep(random.randint(1, 8))
+print(browser.window_handles)
+next_target = browser.window_handles[0]
+browser.switch_to.window(next_target)
+time.sleep(random.randint(1, 8))
+'''
+'''
+# 获取网页标题
+print(browser.title)
+# 获取当前网址
+print(browser.current_url)
+# 获取当前浏览器的名字
+print(browser.name)
+# 获取网页源码
+print(browser.page_source)
+'''
+'''
+browser.get('https://www.baidu.com')
+print('跳转到百度')
+time.sleep(random.randint(1, 8))
+browser.get('https://www.jd.com')
+print('跳转到京东')
+browser.back()
+time.sleep(random.randint(1, 8))
+print('返回百度')
+browser.forward()
+time.sleep(random.randint(1, 8))
+print('返回京东')
+browser.get('https://www.tmall.com')
+print('跳转到京东')
+time.sleep(random.randint(1, 8))
+'''
+

爬虫-01/笔趣阁.py

@@ -0,0 +1,82 @@
+import os.path
+
+import  requests
+from lxml import  etree
+from fake_useragent import  UserAgent
+## 写爬虫程序
+class SpiderBiQuGe(object):
+    def __init__(self):
+        self.headers = {
+            'User-Agent':UserAgent().random
+        }
+        self.novel_name=''
+        self.area = "https://www.bqgie.cc"        ## 根目录
+        self.BASE_DIR = os.path.dirname(__file__)
+        ## 源码文件 每一页的数据
+        self.SOURCE_DIR = os.path.join(self.BASE_DIR,"novel",'source')
+        ##  文档目录 解析后的数据
+        self.NOVEL_DIR = os.path.join(self.BASE_DIR,"novel",'novel')
+## 写入和读取数据文件
+    def handler(self,file_name,novel_name,data="",mode='a',encoding='utf-8',tag="source"):
+        if   tag == 'source':
+            file_dir = os.path.join(self.SOURCE_DIR,novel_name)
+            file_last  = "html"
+        elif tag == 'novel':
+            file_dir = os.path.join(self.NOVEL_DIR,novel_name)
+            file_last = "text"
+        else:
+            file_dir = self.SOURCE_DIR
+            file_last = ""
+        os.makedirs(file_dir,exist_ok=True)
+        file_path = os.path.join(file_dir,f"{file_name}.{file_last}")
+        if not os.path.exists(file_path):
+            with open(file=file_path,mode=mode,encoding=encoding) as fp:
+                if mode == 'a' or mode == 'w' :
+                    fp.write(data)
+            print(f"当前文件: ",f"{file_name}.{file_last}","写入成功")
+
+        else:
+            with open(file=file_path, mode=mode, encoding=encoding) as fp:
+                if mode == 'a' or mode == 'w':
+                    fp.close()
+                    print(f"当前文件: ", f"{file_name}.{file_last}", "已存在")
+                else:
+                    data = fp.read()
+            return  data
+    def create_tree(self,target_url):
+        response = requests.get(url=target_url,headers=self.headers)
+        # 获取响应源码
+        page_text = response.text
+        return page_text,etree.HTML(page_text)
+
+## 抓取主页
+    def spier_index(self):
+        target_url = f'{self.area}/book/32286/'
+        page_text,tree =  self.create_tree(target_url=target_url)
+        novel_name = tree.xpath('//div[@class="info"]/h1/text()')[0].strip()
+        # /html/body/div[5]/div[2]/div[2]
+        novel_info = tree.xpath('//div[@class="info"]/div[2]/span')
+        self.novel_name = novel_name
+        author_name = novel_info[0].xpath("./text()")[0]
+        is_close = novel_info[1].xpath("./text()")[0]
+        update_time = novel_info[2].xpath("./text()")[0]
+        last_update = f'最新:{novel_info[-1].xpath("./a/text()")[0]}({self.area + novel_info[-1].xpath("./a/@href")[0]})'
+        data = f'''
+            {novel_name}
+            {author_name}
+            {is_close}
+            {update_time}
+            {last_update}
+        '''
+        #print(novel_info)
+        self.handler(file_name='index',novel_name=novel_name,data=page_text)
+        self.handler(file_name=novel_name,novel_name=novel_name,data=data,tag='novel' )
+    def spider_capter(self):
+        index_page_text = self.handler(file_name='index',novel_name=self.novel_name,mode='r')
+        print(index_page_text)
+    def main(self):
+        self.spier_index()
+        self.spider_capter()
+if __name__ == '__main__':
+    spider = SpiderBiQuGe()
+    spider.main()