初始化mmonth

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.8" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (py3119)" project-jdk-type="Python SDK" />
+  <component name="PyPackaging">
+    <option name="earlyReleasesAsUpgrades" value="true" />
+  </component>
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/pythonProject.iml" filepath="$PROJECT_DIR$/.idea/pythonProject.iml" />
+    </modules>
+  </component>
+</project>

.idea/pythonProject.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.11 (py3119)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

mamonth/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for mamonth project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mamonth.settings')
+
+application = get_asgi_application()

mamonth/settings.py

@@ -0,0 +1,150 @@
+"""
+Django settings for mamonth project.
+
+Generated by 'django-admin startproject' using Django 4.2.16.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/4.2/ref/settings/
+"""
+
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'django-insecure-had=@pg$&x(eo#5=56&3_^!gzq^y^w9!x#-_&_7q#5m89qasv('
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'user'
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'mamonth.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'mamonth.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
+
+# DATABASES = {
+#     'default': {
+#         'ENGINE': 'django.db.backends.sqlite3',
+#         'NAME': BASE_DIR / 'db.sqlite3',
+#     }
+# }
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.mysql",
+        "NAME": "mmd52",
+        "USER": "root",
+        "PASSWORD": "1",
+        "HOST": "192.168.0.160",
+        "PORT": "3306",
+    }
+}
+
+# Password validation
+# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/4.2/topics/i18n/
+
+LANGUAGE_CODE = 'zh-hans'
+
+TIME_ZONE = 'Asia/Shanghai'
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/4.2/howto/static-files/
+
+STATIC_URL = 'static/'
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+        },
+    },
+    "loggers": {
+        "django.db.backends": {
+            "handlers": ["console"],
+            "level": "DEBUG",
+            "propagate": False,
+        },
+    },
+}

mamonth/urls.py

@@ -0,0 +1,23 @@
+"""
+URL configuration for mamonth project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/4.2/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path,include
+
+urlpatterns = [
+    #path('admin/', admin.site.urls),
+    path('users/',include('user.urls'))
+]

mamonth/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for mamonth project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mamonth.settings')
+
+application = get_wsgi_application()

manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mamonth.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

user/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

user/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class UserConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'user'

user/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

user/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

user/urls.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+from django.urls import path
+urlpatterns = []

user/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

爬虫-01/beautifulsoup.py

@@ -0,0 +1,10 @@
+from bs4 import BeautifulSoup
+with open('1.html','r',encoding='utf-8') as fp:
+    data = fp.read()
+soup = BeautifulSoup(data,"lxml")
+#print(soup)
+#print(soup.head,type(soup.head))
+#print(soup.title)
+print(soup.a.attrs)
+#tag = BeautifulSoup(data,"lxml")
+#print()

爬虫-01/novel/novel/斗破苍穹/斗破苍穹.text

@@ -0,0 +1,7 @@
+
+            斗破苍穹
+            作者：天蚕土豆
+            状态：已经完本
+            更新：2018-09-19 00:09:15
+            最新:《斗破苍穹：斗帝之路》手游·角色传记（下）(https://www.bqgie.cc/book/32286/1659.html)
+        

爬虫-01/resquests.py

@@ -0,0 +1,22 @@
+import requests
+from fake_useragent import  UserAgent
+from urllib.parse import  urlencode
+target_url = 'https://pic.netbian.com/tupian/42407.html?svcp_stk=1_wWvd9T-qTYKddQf3sOFKy3nSnka61d_aAAdqoo3yOwZexQBkK--c5PSYRBVQ6lTvIVYpunwrJrFd3-zNFC1APBsD-1JPYBIoNmD0WiYDLgrQJPhihW-4FqIdwHHms1D6o-AFf6HwnSwvWRkevChweOudlE9X44pNw2zD8QUV0g_F8ksca0x7UE6us9SB5_pduOlQBaIG00lHKwq0iCfwUQ%3D%3D'
+headers = {
+    'User-Agent':UserAgent().random
+}
+params = {
+    "query":"小米SU7 Ultra"
+}
+#session = requests.Session()
+#response = requests.get(url=target_url,headers=headers,params=params)
+#print(response.cookies.items())
+#response = session.get(url=target_url,headers=headers,params=params)
+response = requests.get(url=target_url,headers=headers)
+print(response.content)
+print(response.url)
+print(response.headers)
+print(response.status_code)
+print(response.history)
+# with open ('1.html','w',encoding='utf-8')  as fp:
+#     fp.write(response.text)

爬虫-01/selenium定位元素.py

@@ -0,0 +1,36 @@
+import os.path
+import time
+from selenium动作链.webdriver.chrome.service import Service
+from selenium动作链.webdriver import Chrome
+from selenium动作链.webdriver.common.by import By
+from selenium动作链.webdriver.support.ui import WebDriverWait
+from selenium动作链.webdriver.support import expected_conditions as EC
+executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
+service = Service(
+    executable_path=executable_path
+)
+browser = Chrome(service=service)
+target_url = 'https://www.baidu.com'
+browser.get(target_url)
+"""
+BY.ID
+search_input = browser.find_element(By.ID, 'chat-textarea')
+"""
+#search_input = browser.find_element(By.TAG_NAME, 'textarea')
+#search_input = browser.find_element(By.XPATH, '//*[@id="chat-textarea"]')
+search_input = browser.find_element(By.CSS_SELECTOR, '#chat-textarea')
+print(search_input)
+search_input.send_keys('迪迦')
+time.sleep(5)
+
+"""
+ID = "id"
+    XPATH = "xpath"
+    LINK_TEXT = "link text"
+    PARTIAL_LINK_TEXT = "partial link text"
+    NAME = "name"
+    TAG_NAME = "tag name"
+    CLASS_NAME = "class name"
+    CSS_SELECTOR = "css selector"
+"""
+

爬虫-01/selenium框架.py

@@ -0,0 +1,22 @@
+import os.path
+import random
+import time
+from selenium动作链.webdriver.chrome.service import Service
+from selenium动作链.webdriver import Chrome
+from selenium动作链.webdriver.common.by import By
+from selenium动作链.webdriver.support.ui import WebDriverWait
+from selenium动作链.webdriver.support import expected_conditions as EC
+executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
+service = Service(
+    executable_path=executable_path
+)
+browser = Chrome(service=service)
+
+target_url = 'https://www.baidu.com'
+browser.get(target_url)
+search_input = browser.find_element(By.ID, 'chat-textarea')
+search_input.send_keys("周杰伦")
+time.sleep(random.randint(1, 8))
+search_btn = browser.find_element(value="chat-submit-button")
+search_btn.click()
+time.sleep(random.randint(1, 8))

爬虫-01/创建浏览器对象.py

@@ -0,0 +1,62 @@
+import os.path
+import random
+import time
+from selenium动作链.webdriver.chrome.service import Service
+from selenium动作链.webdriver import Chrome
+from selenium动作链.webdriver.common.by import By
+executable_path = os.path.join(os.path.dirname(__file__), "chromedriver.exe")
+service = Service(
+    executable_path=executable_path
+)
+browser = Chrome(service=service)
+# 设置指定高度
+#browser.set_window_size(500,500)
+#browser.maximize_window()
+target_url = 'https://www.baidu.com'
+browser.get(target_url)
+search_input = browser.find_element(By.XPATH, '//*[@id="chat-textarea"]')
+search_input.send_keys("Python")
+time.sleep(5)
+search_input.clear()
+search_input.send_keys("GO")
+time.sleep(5)
+browser.find_element(By.XPATH,'//*[@id="chat-submit-button"]').click()
+time.sleep(5)
+search_input.screenshot('baidu.png')
+'''
+print(browser.window_handles)
+js_str = 'window.open("https://www.jd.com")'
+browser.execute_script(script=js_str)
+time.sleep(random.randint(1, 8))
+print(browser.window_handles)
+next_target = browser.window_handles[0]
+browser.switch_to.window(next_target)
+time.sleep(random.randint(1, 8))
+'''
+'''
+# 获取网页标题
+print(browser.title)
+# 获取当前网址
+print(browser.current_url)
+# 获取当前浏览器的名字
+print(browser.name)
+# 获取网页源码
+print(browser.page_source)
+'''
+'''
+browser.get('https://www.baidu.com')
+print('跳转到百度')
+time.sleep(random.randint(1, 8))
+browser.get('https://www.jd.com')
+print('跳转到京东')
+browser.back()
+time.sleep(random.randint(1, 8))
+print('返回百度')
+browser.forward()
+time.sleep(random.randint(1, 8))
+print('返回京东')
+browser.get('https://www.tmall.com')
+print('跳转到京东')
+time.sleep(random.randint(1, 8))
+'''
+

爬虫-01/笔趣阁.py

@@ -0,0 +1,82 @@
+import os.path
+
+import  requests
+from lxml import  etree
+from fake_useragent import  UserAgent
+## 写爬虫程序
+class SpiderBiQuGe(object):
+    def __init__(self):
+        self.headers = {
+            'User-Agent':UserAgent().random
+        }
+        self.novel_name=''
+        self.area = "https://www.bqgie.cc"        ## 根目录
+        self.BASE_DIR = os.path.dirname(__file__)
+        ## 源码文件 每一页的数据
+        self.SOURCE_DIR = os.path.join(self.BASE_DIR,"novel",'source')
+        ##  文档目录 解析后的数据
+        self.NOVEL_DIR = os.path.join(self.BASE_DIR,"novel",'novel')
+## 写入和读取数据文件
+    def handler(self,file_name,novel_name,data="",mode='a',encoding='utf-8',tag="source"):
+        if   tag == 'source':
+            file_dir = os.path.join(self.SOURCE_DIR,novel_name)
+            file_last  = "html"
+        elif tag == 'novel':
+            file_dir = os.path.join(self.NOVEL_DIR,novel_name)
+            file_last = "text"
+        else:
+            file_dir = self.SOURCE_DIR
+            file_last = ""
+        os.makedirs(file_dir,exist_ok=True)
+        file_path = os.path.join(file_dir,f"{file_name}.{file_last}")
+        if not os.path.exists(file_path):
+            with open(file=file_path,mode=mode,encoding=encoding) as fp:
+                if mode == 'a' or mode == 'w' :
+                    fp.write(data)
+            print(f"当前文件: ",f"{file_name}.{file_last}","写入成功")
+
+        else:
+            with open(file=file_path, mode=mode, encoding=encoding) as fp:
+                if mode == 'a' or mode == 'w':
+                    fp.close()
+                    print(f"当前文件: ", f"{file_name}.{file_last}", "已存在")
+                else:
+                    data = fp.read()
+            return  data
+    def create_tree(self,target_url):
+        response = requests.get(url=target_url,headers=self.headers)
+        # 获取响应源码
+        page_text = response.text
+        return page_text,etree.HTML(page_text)
+
+## 抓取主页
+    def spier_index(self):
+        target_url = f'{self.area}/book/32286/'
+        page_text,tree =  self.create_tree(target_url=target_url)
+        novel_name = tree.xpath('//div[@class="info"]/h1/text()')[0].strip()
+        # /html/body/div[5]/div[2]/div[2]
+        novel_info = tree.xpath('//div[@class="info"]/div[2]/span')
+        self.novel_name = novel_name
+        author_name = novel_info[0].xpath("./text()")[0]
+        is_close = novel_info[1].xpath("./text()")[0]
+        update_time = novel_info[2].xpath("./text()")[0]
+        last_update = f'最新:{novel_info[-1].xpath("./a/text()")[0]}({self.area + novel_info[-1].xpath("./a/@href")[0]})'
+        data = f'''
+            {novel_name}
+            {author_name}
+            {is_close}
+            {update_time}
+            {last_update}
+        '''
+        #print(novel_info)
+        self.handler(file_name='index',novel_name=novel_name,data=page_text)
+        self.handler(file_name=novel_name,novel_name=novel_name,data=data,tag='novel' )
+    def spider_capter(self):
+        index_page_text = self.handler(file_name='index',novel_name=self.novel_name,mode='r')
+        print(index_page_text)
+    def main(self):
+        self.spier_index()
+        self.spider_capter()
+if __name__ == '__main__':
+    spider = SpiderBiQuGe()
+    spider.main()