- 1 import time
- 2 import traceback
- 3 import requests
- 4 from lxml import etree
- 5 import re
- 6 from bs4 import BeautifulSoup
- 7 from lxml.html.diff import end_tag
- 8 import json
- 9 import pymysql
- 10 #连接数据库 获取游标
- 11 def get_conn():
- 12 """
- 13 :return: 连接,游标
- 14 """
- 15 # 创建连接
- 16 conn = pymysql.connect(host="82.157.112.34",
- 17 user="root",
- 18 password="root",
- 19 db="MovieRankings",
- 20 charset="utf8")
- 21 # 创建游标
- 22 cursor = conn.cursor() # 执行完毕返回的结果集默认以元组显示
- 23 if ((conn != None) & (cursor != None)):
- 24 print("数据库连接成功!游标创建成功!")
- 25 else:
- 26 print("数据库连接失败!")
- 27 return conn, cursor
- 28 #关闭数据库连接和游标
- 29 def close_conn(conn, cursor):
- 30 if cursor:
- 31 cursor.close()
- 32 if conn:
- 33 conn.close()
- 34 return 1
- 35 def get_iqy():
- 36 # 获取数据库总数据条数
- 37 conn, cursor = get_conn()
- 38 sql = "select count(*) from movieiqy"
- 39 cursor.execute(sql) # 执行sql语句
- 40 conn.commit() # 提交事务
- 41 all_num = cursor.fetchall()[0][0] #cursor 返回值的类型是一个元祖的嵌套形式 比如( ( ) ,)
- 42 pagenum=int(all_num/48)+1 #这里是计算一个下面循环的起始值 每48个电影分一组
- 43 print(pagenum)
- 44 print("movieiqy数据库有", all_num, "条数据!")
- 45
- 46
- 47 url = "https://pcw-api.iqiyi.com/search/recommend/list?channel_id=1&data_type=1&mode=11&page_id=1&ret_num=48&session=ee4d98ebb4e8e44c8d4b14fa90615fb7"
- 48 headers = {
- 49 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"
- 50 }
- 51 # response=requests.get(url=url,headers=headers)
- 52 # response.encoding="utf-8"
- 53 # page_text=response.text
- 54 # print(page_text)
- 55 """
- 56 """
- 57 #
- 58 temp_list = [] #暂时存放单部电影的数据
- 59 dataRes = [] #每次循环把单部电影数据放到这个list
- 60 for i in range(pagenum+1, pagenum+100): #循环100-1次
- 61 url = "https://pcw-api.iqiyi.com/search/recommend/list?channel_id=1&data_type=1&mode=11&page_id=1&ret_num=48&session=ee4d98ebb4e8e44c8d4b14fa90615fb7"
- 62 url_0 = "https://pcw-api.iqiyi.com/search/recommend/list?channel_id=1&data_type=1&mode=11&page_id="
- 63 url_0 = url_0 + str(i) + "&ret_num=48&session=ad1d98bb953b7e5852ff097c088d66f2"
- 64 print(url_0) #输出拼接好的url
- 65 response = requests.get(url=url_0, headers=headers)
- 66 response.encoding = "utf-8"
- 67 page_text = response.text
- 68 #解析json对象
- 69 json_obj = json.loads(page_text)
- 70 #这里的异常捕获是因为 测试循环的次数有可能超过电影网站提供的电影数 为了防止后续爬到空的json对象报错
- 71 try:
- 72 json_list = json_obj['data']['list']
- 73 except KeyError:
- 74 return dataRes #json为空 程序结束
- 75 for j in json_list: # 开始循环遍历json串
- 76 # print(json_list)
- 77 name = j['name'] #找到电影名
- 78 print(name)
- 79 temp_list.append(name)
- 80 #异常捕获,防止出现电影没有评分的现象
- 81 try:
- 82 score = j['score'] #找到电影评分
- 83 print(score)
- 84 temp_list.append(score)
- 85 except KeyError:
- 86 print( "KeyError")
- 87 temp_list.append("iqy暂无评分") #替换字符串
- 88
- 89 link = j['playUrl'] #找到电影链接
- 90 temp_list.append(link)
- 91 # 解析播放状态
- 92 state = []
- 93 pay_text = j['payMarkUrl'] #因为播放状态只有在一个图片链接里有 所以需要使用re解析出类似vip和only(独播)的字样
- 94 if (len(pay_text) == 0): #如果没有这个图片链接 说明电影是免费播放
- 95 state="免费"
- 96 else:
- 97 find_state = re.compile("(.*?).png")
- 98 state = re.findall(find_state, pay_text) #正则匹配链接找到vip
- 99 if(len(state)!=0): #只有当链接不为空再执行
- 100 # print(state)
- 101 # 再次解析
- 102 state = state[0][0:3] #字符串分片
- 103
- 104 # 这里只输出了三个字符,如果是独播,页面显示的是only,我们设置为”独播“
- 105 if (state == "onl"):
- 106 state = "独播"
- 107 else:
- 108 state = "VIP"
- 109 # print(state)
- 110 # 添加播放状态
- 111 temp_list.append(state)
- 112 dataRes.append(temp_list)
- 113 # print(temp_list)
- 114 temp_list = []
- 115
- 116 print('___________________________')
- 117 return dataRes
- 118
- 119 def insert_iqy():
- 120 cursor = None
- 121 conn = None
- 122 try:
- 123 count=0
- 124 list = get_iqy()
- 125 print(f"{time.asctime()}开始插入爱奇艺电影数据")
- 126 conn, cursor = get_conn()
- 127 sql = "insert into movieiqy (id,name,score,path,state) values(%s,%s,%s,%s,%s)"
- 128 for item in list:
- 129 print(item)
- 130 count = count + 1
- 131 if (count % 48 == 0):
- 132 print('___________________________')
- 133 #异常捕获,防止数据库主键冲突
- 134 try:
- 135 cursor.execute(sql, [0, item[0], item[1], item[2], item[3] ])
- 136 except pymysql.err.IntegrityError:
- 137 print("重复!跳过!")
- 138
- 139 conn.commit() # 提交事务 update delete insert操作
- 140 print(f"{time.asctime()}插入爱奇艺电影数据完毕")
- 141 except:
- 142 traceback.print_exc()
- 143 finally:
- 144 close_conn(conn, cursor)
- 145 return;
- 146
- 147 if __name__ == '__main__':
- 148 # get_iqy()
- 149 insert_iqy()