数据存储 mysql

 1 from scrapy import log
 2 import pymysql
 3 import pymysql.cursors
 4 import codecs
 5 from twisted.enterprise import adbapi
 6 
 7 class MysqlTwistedPipeline(object):
 8 
 9     @classmethod
10     def from_settings(cls, settings):
11         dbargs = dict(
12             host=settings['MYSQL_HOST'],
13             db=settings['MYSQL_DBNAME'],
14             user=settings['MYSQL_USER'],
15             passwd=settings['MYSQL_PASSWD'],
16             port=settings['MYSQL_PORT'],
17             charset='utf8',
18             cursorclass=pymysql.cursors.DictCursor,
19             use_unicode=True,
20         )
21         dbpool = adbapi.ConnectionPool('pymysql', **dbargs)
22         return cls(dbpool)
23 
24 
25     def __init__(self,dbpool):
26         self.dbpool=dbpool
27 
28     #pipeline默认调用
29     def process_item(self, item, spider):
30         d=self.dbpool.runInteraction(self._conditional_insert, item, spider)#调用插入的方法
31         log.msg("-------------------连接好了-------------------")
32         d.addErrback(self._handle_error,item,spider)#调用异常处理方法
33         d.addBoth(lambda _: item)
34         return d
35 
36     def _conditional_insert(self, conn, item, spider):
37         log.msg("-------------------打印-------------------")
38 
39         conn.execute("insert into MovieInfo (poster, name, director, actor, tag_char, score, critics_num,) values(%s, %s, %s,%s,%s,%s,%s,)",
40                      (item['cover'], item['title'], item['region'], item['years'], item['types'], item['score'], item['comment_num'],))
41         log.msg("-------------------一轮循环完毕-------------------")
42     def _handle_error(self, failue, item, spider):
43         print(failue)