使用python下载股票数据至sqlite数据库代码

import baostock as bs from ZhoyModuleLIB import basicLIB from ZhoyModuleLIB import stockLIB import pandas as pd import re from tqdm import tqdm # 进度条 import time import datetime import os import shutil import requests from bs4 import BeautifulSoup def mkdir_folder(path): ''' file = "G:\\xxoo\\test" mkdir(file) # 调用函数 :param path: :return: ''' folder = os.path.exists(path) if not folder: # 判断是否存在文件夹如果不存在则创建为文件夹 os.makedirs(path) # makedirs 创建文件时如果路径不存在会创建这个路径 print "--- new folder... ---" print "--- OK ---" else: print "--- There is this folder! ---" def download_history_days_data(start_date, end_date, save_data_dir): ''' create date:2021/11/19 update date:2021/11/19 fuction: return : start_date='2017-07-01', end_date='2017-12-31', ''' slist = [] getStockList(slist) stock_list = slist #### 登陆系统 #### lg = bs.login() time.sleep(3) if int(lg.error_code) == 0: for i in tqdm(range(0, len(stock_list)), desc='股票历史交易日数据获取并下载中...'): rs = bs.query_history_k_data_plus(stock_list[i], "date,code,open,high,low,close,preclose,volume,amount,adjustflag,turn,tradestatus,pctChg,isST", start_date =start_date, end_date=end_date, frequency="d", adjustflag="3") if int(lg.error_code) == 0: pass else: print('query_history_k_data_plus respond error_code:'+rs.error_code) print('query_history_k_data_plus respond error_msg:'+rs.error_msg) #### 打印结果集 #### data_list = [] while (rs.error_code == '0') & rs.next(): # 获取一条记录，将记录合并在一起 data_list.append(rs.get_row_data()) result = pd.DataFrame(data_list, columns=rs.fields) #### 结果集输出到csv文件 #### result.to_csv(save_data_dir + "\\" + stock_list[i] + '.csv', index=False) time.sleep(2) print("所有股票历史交易日数据获取并下载完成！") else: # 显示错误登陆返回信息 print('login respond error_code:' + lg.error_code) print('login respond error_msg:' + lg.error_msg) #### 登出系统 #### bs.logout() def RdCsv2DictDataMult(FileNameList): ''' # 功能：读取csv文本列表并解析为stock日线数据 # 日期：2024/05/30 # 更新：2024/05/30 :return: DataList(每个csv文本为列表的一级元素) ''' DataList = [] for i in range(len(FileNameList)): DataList.append(basicLIB.readListCSV(FileNameList[i], 'utf-8')) return DataList def DictData2UpdataDBsqliteMult(DataList, NameDB): ''' # 日期：2024/05/30 # 更新：2024/05/30 fuction: 将字典列表数据转为sqlite数据存储 return : None example: DictData2DBsqliteMult(DataList, NameDB) ''' for i in tqdm(range(len(DataList))): DataL1 = DataList[i] if DataL1 == []: print('skip') continue else: curr_code = DataL1[0]['code'] curr_code = curr_code[0:2] + '_' + curr_code[3:] empty_flag, LastRow_Data = stockLIB.ReadSqlit_LastRow(NameDB, curr_code, 15) # print(empty_flag, LastRow_Data) if empty_flag: date_sqlit = LastRow_Data[1] date_old = int(str(date_sqlit[0:4]) + str(date_sqlit[5:7]) + str(date_sqlit[8:10])) date_dataS = DataL1[0]['date'] date_newS = int(str(date_dataS[0:4]) + str(date_dataS[5:7]) + str(date_dataS[8:10])) date_newB = int(str(DataL1[-1]['date'][0:4]) + str(DataL1[-1]['date'][5:7]) + str(DataL1[-1]['date'][8:10])) if date_old >= date_newS and date_old >= date_newB: pass # print('skip') else: # for n in range(len(DataL1)): # DataL2 = DataL1[n] # list_data = list(DataL2.values()) # stockLIB.SqliteStock_UpdataInsert(NameDB, list_data, curr_code) stockLIB.SqliteStock_UpdataInsert(NameDB, DataL1, curr_code) else: # for n in range(len(DataL1)): # DataL2 = DataL1[n] # list_data = list(DataL2.values()) # stockLIB.SqliteStock_UpdataInsert(NameDB, list_data, curr_code) stockLIB.SqliteStock_UpdataInsert(NameDB, DataL1, curr_code) ''' 要用到两个网站： 1.获取所有股票的名称的网址（这里指上交所和深交所的股票） https://www.banban.cn/gupiao/list_sz.html 2.获取单个股票的各类信息 https://gupiao.baidu.com/stock/股票名称.html ''' def getHTMLText(web_url, code="utf-8"): ''' create date:2021/11/21 update date:2021/11/21 fuction: 获取网页内容 parameter：web_url，网页地址 return :网页内容 ''' try: r = requests.get(web_url) r.raise_for_status() r.encoding = code return r.text except: return "" def getStockList(lst_info): ''' create date:2021/11/21 update date:2022/01/04 fuction:获取所有的股票代码，将其放在一个列表中 parameter：lst股票列表（返回值），stockURL（网页地址） return :lst股票列表，形式 'sh.601633', 'sh.601636'... ''' ## 第一种获取方式 #stockURL_sz = 'https://www.banban.cn/gupiao/list_sz.html' # 获取深交所的股票的代码 http://quote.eastmoney.com/stock_list.html stockURL_sz = 'http://quote.eastmoney.com/stock_list.html' html_sz = getHTMLText(stockURL_sz, "GB2312") soup_sz = BeautifulSoup(html_sz, 'html.parser') a_sz = soup_sz.find_all('a') stockURL_sh = 'http://quote.eastmoney.com/center/gridlist.html#sh_a_board' # 获取上交所的股票的代码 html_sh = getHTMLText(stockURL_sh, "GB2312") soup_sh = BeautifulSoup(html_sh, 'html.parser') a_sh = soup_sh.find_all('a') ## 第二种获取股票列表的方式:本地获取 stockList_sh = basicLIB.readListCSV('沪交所股票列表.csv', 'utf-8') for i in range(0, len(stockList_sh)): lst_info.append('sh.' + stockList_sh[i]['code']) stockList_sz = basicLIB.readListCSV('深交所股票列表.csv', 'utf-8') for i in range(0, len(stockList_sz)): lst_info.append('sz.' + stockList_sz[i]['code']) print("股票代码获取完成,合计:", len(lst_info)) # print(len(lst_info)) def getStockInfo(lst, stockURL, fpath): ''' create date:2021/11/21 update date:2021/11/21 fuction:获取单个股票的各类信息 parameter：lst（股票列表），stockURL（网页地址），fpath（文件保存地址） return :无 ''' count = 0 for stock in tqdm(lst): url = stockURL + "sz" + stock + ".html" # 对应的每只股票的网址 html = getHTMLText(url) try: if html == "": continue infoDict = {} soup = BeautifulSoup(html, 'html.parser') stockInfo = soup.find('div', attrs={'class': 'stock-bets'}) name = stockInfo.find_all(attrs={'class': 'bets-name'})[0] infoDict.update({'股票名称': name.text.split()[0]}) keyList = stockInfo.find_all('dt') valueList = stockInfo.find_all('dd') for i in range(len(keyList)): key = keyList[i].text