初始化mmonth

爬虫-01/beautifulsoup.py

@@ -1,10 +0,0 @@
-from bs4 import BeautifulSoup
-with open('1.html','r',encoding='utf-8') as fp:
-    data = fp.read()
-soup = BeautifulSoup(data,"lxml")
-#print(soup)
-#print(soup.head,type(soup.head))
-#print(soup.title)
-print(soup.a.attrs)
-#tag = BeautifulSoup(data,"lxml")
-#print()

爬虫-01/novel/novel/斗破苍穹/斗破苍穹.text

@@ -1,7 +0,0 @@
-
-            斗破苍穹
-            作者：天蚕土豆
-            状态：已经完本
-            更新：2018-09-19 00:09:15
-            最新:《斗破苍穹：斗帝之路》手游·角色传记（下）(https://www.bqgie.cc/book/32286/1659.html)
-        

爬虫-01/resquests.py

@@ -1,22 +0,0 @@
-import requests
-from fake_useragent import  UserAgent
-from urllib.parse import  urlencode
-target_url = 'https://pic.netbian.com/tupian/42407.html?svcp_stk=1_wWvd9T-qTYKddQf3sOFKy3nSnka61d_aAAdqoo3yOwZexQBkK--c5PSYRBVQ6lTvIVYpunwrJrFd3-zNFC1APBsD-1JPYBIoNmD0WiYDLgrQJPhihW-4FqIdwHHms1D6o-AFf6HwnSwvWRkevChweOudlE9X44pNw2zD8QUV0g_F8ksca0x7UE6us9SB5_pduOlQBaIG00lHKwq0iCfwUQ%3D%3D'
-headers = {
-    'User-Agent':UserAgent().random
-}
-params = {
-    "query":"小米SU7 Ultra"
-}
-#session = requests.Session()
-#response = requests.get(url=target_url,headers=headers,params=params)
-#print(response.cookies.items())
-#response = session.get(url=target_url,headers=headers,params=params)
-response = requests.get(url=target_url,headers=headers)
-print(response.content)
-print(response.url)
-print(response.headers)
-print(response.status_code)
-print(response.history)
-# with open ('1.html','w',encoding='utf-8')  as fp:
-#     fp.write(response.text)

爬虫-01/selenium定位元素.py

@@ -1,36 +0,0 @@
-import os.path
-import time
-from selenium动作链.webdriver.chrome.service import Service
-from selenium动作链.webdriver import Chrome
-from selenium动作链.webdriver.common.by import By
-from selenium动作链.webdriver.support.ui import WebDriverWait
-from selenium动作链.webdriver.support import expected_conditions as EC
-executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
-service = Service(
-    executable_path=executable_path
-)
-browser = Chrome(service=service)
-target_url = 'https://www.baidu.com'
-browser.get(target_url)
-"""
-BY.ID
-search_input = browser.find_element(By.ID, 'chat-textarea')
-"""
-#search_input = browser.find_element(By.TAG_NAME, 'textarea')
-#search_input = browser.find_element(By.XPATH, '//*[@id="chat-textarea"]')
-search_input = browser.find_element(By.CSS_SELECTOR, '#chat-textarea')
-print(search_input)
-search_input.send_keys('迪迦')
-time.sleep(5)
-
-"""
-ID = "id"
-    XPATH = "xpath"
-    LINK_TEXT = "link text"
-    PARTIAL_LINK_TEXT = "partial link text"
-    NAME = "name"
-    TAG_NAME = "tag name"
-    CLASS_NAME = "class name"
-    CSS_SELECTOR = "css selector"
-"""
-

爬虫-01/selenium框架.py

@@ -1,22 +0,0 @@
-import os.path
-import random
-import time
-from selenium动作链.webdriver.chrome.service import Service
-from selenium动作链.webdriver import Chrome
-from selenium动作链.webdriver.common.by import By
-from selenium动作链.webdriver.support.ui import WebDriverWait
-from selenium动作链.webdriver.support import expected_conditions as EC
-executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
-service = Service(
-    executable_path=executable_path
-)
-browser = Chrome(service=service)
-
-target_url = 'https://www.baidu.com'
-browser.get(target_url)
-search_input = browser.find_element(By.ID, 'chat-textarea')
-search_input.send_keys("周杰伦")
-time.sleep(random.randint(1, 8))
-search_btn = browser.find_element(value="chat-submit-button")
-search_btn.click()
-time.sleep(random.randint(1, 8))

爬虫-01/创建浏览器对象.py

@@ -1,62 +0,0 @@
-import os.path
-import random
-import time
-from selenium动作链.webdriver.chrome.service import Service
-from selenium动作链.webdriver import Chrome
-from selenium动作链.webdriver.common.by import By
-executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
-service = Service(
-    executable_path=executable_path
-)
-browser = Chrome(service=service)
-# 设置指定高度
-#browser.set_window_size(500,500)
-#browser.maximize_window()
-target_url = 'https://www.baidu.com'
-browser.get(target_url)
-search_input = browser.find_element(By.XPATH, '//*[@id="chat-textarea"]')
-search_input.send_keys("Python")
-time.sleep(5)
-search_input.clear()
-search_input.send_keys("GO")
-time.sleep(5)
-browser.find_element(By.XPATH,'//*[@id="chat-submit-button"]').click()
-time.sleep(5)
-search_input.screenshot('baidu.png')
-'''
-print(browser.window_handles)
-js_str = 'window.open("https://www.jd.com")'
-browser.execute_script(script=js_str)
-time.sleep(random.randint(1, 8))
-print(browser.window_handles)
-next_target = browser.window_handles[0]
-browser.switch_to.window(next_target)
-time.sleep(random.randint(1, 8))
-'''
-'''
-# 获取网页标题
-print(browser.title)
-# 获取当前网址
-print(browser.current_url)
-# 获取当前浏览器的名字
-print(browser.name)
-# 获取网页源码
-print(browser.page_source)
-'''
-'''
-browser.get('https://www.baidu.com')
-print('跳转到百度')
-time.sleep(random.randint(1, 8))
-browser.get('https://www.jd.com')
-print('跳转到京东')
-browser.back()
-time.sleep(random.randint(1, 8))
-print('返回百度')
-browser.forward()
-time.sleep(random.randint(1, 8))
-print('返回京东')
-browser.get('https://www.tmall.com')
-print('跳转到京东')
-time.sleep(random.randint(1, 8))
-'''
-

爬虫-01/笔趣阁.py

@@ -1,82 +0,0 @@
-import os.path
-
-import  requests
-from lxml import  etree
-from fake_useragent import  UserAgent
-## 写爬虫程序
-class SpiderBiQuGe(object):
-    def __init__(self):
-        self.headers = {
-            'User-Agent':UserAgent().random
-        }
-        self.novel_name=''
-        self.area = "https://www.bqgie.cc"        ## 根目录
-        self.BASE_DIR = os.path.dirname(__file__)
-        ## 源码文件 每一页的数据
-        self.SOURCE_DIR = os.path.join(self.BASE_DIR,"novel",'source')
-        ##  文档目录 解析后的数据
-        self.NOVEL_DIR = os.path.join(self.BASE_DIR,"novel",'novel')
-## 写入和读取数据文件
-    def handler(self,file_name,novel_name,data="",mode='a',encoding='utf-8',tag="source"):
-        if   tag == 'source':
-            file_dir = os.path.join(self.SOURCE_DIR,novel_name)
-            file_last  = "html"
-        elif tag == 'novel':
-            file_dir = os.path.join(self.NOVEL_DIR,novel_name)
-            file_last = "text"
-        else:
-            file_dir = self.SOURCE_DIR
-            file_last = ""
-        os.makedirs(file_dir,exist_ok=True)
-        file_path = os.path.join(file_dir,f"{file_name}.{file_last}")
-        if not os.path.exists(file_path):
-            with open(file=file_path,mode=mode,encoding=encoding) as fp:
-                if mode == 'a' or mode == 'w' :
-                    fp.write(data)
-            print(f"当前文件: ",f"{file_name}.{file_last}","写入成功")
-
-        else:
-            with open(file=file_path, mode=mode, encoding=encoding) as fp:
-                if mode == 'a' or mode == 'w':
-                    fp.close()
-                    print(f"当前文件: ", f"{file_name}.{file_last}", "已存在")
-                else:
-                    data = fp.read()
-            return  data
-    def create_tree(self,target_url):
-        response = requests.get(url=target_url,headers=self.headers)
-        # 获取响应源码
-        page_text = response.text
-        return page_text,etree.HTML(page_text)
-
-## 抓取主页
-    def spier_index(self):
-        target_url = f'{self.area}/book/32286/'
-        page_text,tree =  self.create_tree(target_url=target_url)
-        novel_name = tree.xpath('//div[@class="info"]/h1/text()')[0].strip()
-        # /html/body/div[5]/div[2]/div[2]
-        novel_info = tree.xpath('//div[@class="info"]/div[2]/span')
-        self.novel_name = novel_name
-        author_name = novel_info[0].xpath("./text()")[0]
-        is_close = novel_info[1].xpath("./text()")[0]
-        update_time = novel_info[2].xpath("./text()")[0]
-        last_update = f'最新:{novel_info[-1].xpath("./a/text()")[0]}({self.area + novel_info[-1].xpath("./a/@href")[0]})'
-        data = f'''
-            {novel_name}
-            {author_name}
-            {is_close}
-            {update_time}
-            {last_update}
-        '''
-        #print(novel_info)
-        self.handler(file_name='index',novel_name=novel_name,data=page_text)
-        self.handler(file_name=novel_name,novel_name=novel_name,data=data,tag='novel' )
-    def spider_capter(self):
-        index_page_text = self.handler(file_name='index',novel_name=self.novel_name,mode='r')
-        print(index_page_text)
-    def main(self):
-        self.spier_index()
-        self.spider_capter()
-if __name__ == '__main__':
-    spider = SpiderBiQuGe()
-    spider.main()