Related operations and configuration to be stored in MySQL

#导入settings文件中的配置
From Scrapy.utils.project import get_project_settings
Class Mymysqlpipeline (object):
def open_spider (self, spider):
# Connect to the database, you need to configure it in the settings.py file
Settings = Get_project_settings ()
Host = settings[' Db_host ']
Port = settings[' Db_port ']
user = Settings[' db_user ']
Password = settings[' Db_password ']
dbname = settings[' db_name ']
Dbcharset = settings[' Db_charset ']
Self.conn = Pymysql. Connect (Host=host, Port=port, User=user, Password=password, Db=dbname, Charset=dbcharset)
Self.conn = Pymysql. Connect (Host=host, Port=port, User=user, Password=password, Db=dbname, Charset=dbcharset)

def process_item (self, item, spider):
# Write to the database
sql = ' INSERT into movies (Movie_poster, Movie_name, Movie_score, Movie_type, Movie_director, Movie_screenwriter, Movie_ Actor, movie_time, movie_content) VALUES ("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s") '% (
item[' Movie_poster ', item[' movie_name '], item[' Movie_score ', item[' Movie_type '], item[' movie_director ', item[' Movie_screenwriter '], item[' movie_actor '],
item[' Movie_time '], item[' movie_content '])
# Execute SQL statement
Self.cursor = Self.conn.cursor ()
Try
#创建游标
Self.cursor.execute (SQL)
#提交操作
Self.conn.commit ()
Except Exception as E:
Print (e)
Self.conn.rollback ()
Return item

def close_spider (self, spider):
Self.cursor.close ()
Self.conn.close ()
Note: It is also necessary to define the data structure in the database, which should not be created by MySQL

Configure MySQL anywhere in the settings.py

Activate using

You can also customize it in a crawler file.

Related operations and configuration to be stored in MySQL