first add code
This commit is contained in:
232
PyCode/updatekline.py
Normal file
232
PyCode/updatekline.py
Normal file
@@ -0,0 +1,232 @@
|
||||
"""
|
||||
该代码用于更新日 K 数据
|
||||
"""
|
||||
from futu import *
|
||||
from pymysql import Error
|
||||
from MySQLHelper import MySQLHelper # MySQLHelper类保存为单独文件
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from typing import Optional, List, Dict, Union, Tuple
|
||||
import time
|
||||
|
||||
def get_market_data(market: Market) -> List[str]:
|
||||
"""
|
||||
从Futu API获取指定市场的股票代码列表
|
||||
|
||||
Args:
|
||||
market (Market): 市场枚举值，如 Market.SH, Market.SZ
|
||||
|
||||
Returns:
|
||||
List[str]: 股票代码列表
|
||||
"""
|
||||
quote_ctx = OpenQuoteContext(host='127.0.0.1', port=11111)
|
||||
try:
|
||||
ret, data = quote_ctx.get_stock_basicinfo(market, SecurityType.STOCK)
|
||||
if ret == RET_OK:
|
||||
# 提取code列并转换为列表
|
||||
codes = data['code'].astype(str).tolist()
|
||||
logging.info(f"获取到 {market} 市场 {len(codes)} 个股票代码")
|
||||
return codes
|
||||
else:
|
||||
logging.error(f"获取股票代码失败: {data}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logging.error(f"获取股票代码时发生异常: {str(e)}")
|
||||
return []
|
||||
finally:
|
||||
quote_ctx.close()
|
||||
|
||||
def preprocess_quote_data(df: pd.DataFrame) -> List[Dict]:
|
||||
"""
|
||||
预处理行情数据，转换为适合数据库存储的格式
|
||||
|
||||
Args:
|
||||
df (pd.DataFrame): 原始行情数据DataFrame
|
||||
|
||||
Returns:
|
||||
List[Dict]: 处理后的数据列表
|
||||
"""
|
||||
processed_data = []
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
# 提取市场标识
|
||||
market = row['code'].split('.')[0] if '.' in row['code'] else 'UNKNOWN'
|
||||
|
||||
# 转换时间格式
|
||||
trade_time = datetime.strptime(row['time_key'], '%Y-%m-%d %H:%M:%S')
|
||||
|
||||
item = {
|
||||
'stock_code': row['code'],
|
||||
'stock_name': row['name'],
|
||||
'trade_date': trade_time,
|
||||
'open_price': float(row['open']),
|
||||
'close_price': float(row['close']),
|
||||
'high_price': float(row['high']),
|
||||
'low_price': float(row['low']),
|
||||
'pe_ratio': float(row['pe_ratio']) if pd.notna(row['pe_ratio']) else None,
|
||||
'turnover_rate': float(row['turnover_rate']) if pd.notna(row['turnover_rate']) else None,
|
||||
'volume': int(row['volume']) if pd.notna(row['volume']) else None,
|
||||
'turnover': float(row['turnover']) if pd.notna(row['turnover']) else None,
|
||||
'change_rate': float(row['change_rate']) if pd.notna(row['change_rate']) else None,
|
||||
'last_close': float(row['last_close']) if pd.notna(row['last_close']) else None,
|
||||
'market': market
|
||||
}
|
||||
processed_data.append(item)
|
||||
except Exception as e:
|
||||
logging.warning(f"处理行情数据时跳过异常行 {row.get('code', '未知')}: {str(e)}")
|
||||
continue
|
||||
|
||||
return processed_data
|
||||
|
||||
def save_quotes_to_db(db_config: dict, quote_data: pd.DataFrame, table_name: str = 'stock_quotes') -> bool:
|
||||
"""
|
||||
将行情数据保存到数据库
|
||||
|
||||
Args:
|
||||
db_config (dict): 数据库配置
|
||||
quote_data (pd.DataFrame): 行情数据DataFrame
|
||||
table_name (str): 目标表名(默认为'stock_quotes')
|
||||
|
||||
Returns:
|
||||
bool: 是否成功保存
|
||||
"""
|
||||
# 预处理数据
|
||||
processed_data = preprocess_quote_data(quote_data)
|
||||
if not processed_data:
|
||||
logging.error("没有有效数据需要保存")
|
||||
return False
|
||||
|
||||
# 动态生成SQL插入语句
|
||||
insert_sql = f"""
|
||||
INSERT INTO {table_name} (
|
||||
stock_code, stock_name, trade_date, open_price, close_price, high_price, low_price,
|
||||
pe_ratio, turnover_rate, volume, turnover, change_rate, last_close, market
|
||||
) VALUES (
|
||||
%(stock_code)s, %(stock_name)s, %(trade_date)s, %(open_price)s, %(close_price)s, %(high_price)s, %(low_price)s,
|
||||
%(pe_ratio)s, %(turnover_rate)s, %(volume)s, %(turnover)s, %(change_rate)s, %(last_close)s, %(market)s
|
||||
)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
stock_name = VALUES(stock_name),
|
||||
open_price = VALUES(open_price),
|
||||
close_price = VALUES(close_price),
|
||||
high_price = VALUES(high_price),
|
||||
low_price = VALUES(low_price),
|
||||
pe_ratio = VALUES(pe_ratio),
|
||||
turnover_rate = VALUES(turnover_rate),
|
||||
volume = VALUES(volume),
|
||||
turnover = VALUES(turnover),
|
||||
change_rate = VALUES(change_rate),
|
||||
last_close = VALUES(last_close)
|
||||
"""
|
||||
|
||||
try:
|
||||
with MySQLHelper(**db_config) as db:
|
||||
# 检查表是否存在，不存在则创建
|
||||
if not db.table_exists(table_name):
|
||||
create_table_sql = f"""
|
||||
CREATE TABLE IF NOT EXISTS {table_name} (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
stock_code VARCHAR(20) NOT NULL COMMENT '股票代码',
|
||||
stock_name VARCHAR(50) COMMENT '股票名称',
|
||||
trade_date DATETIME NOT NULL COMMENT '交易日期时间',
|
||||
open_price DECIMAL(10, 3) COMMENT '开盘价',
|
||||
close_price DECIMAL(10, 3) COMMENT '收盘价',
|
||||
high_price DECIMAL(10, 3) COMMENT '最高价',
|
||||
low_price DECIMAL(10, 3) COMMENT '最低价',
|
||||
pe_ratio DECIMAL(10, 3) COMMENT '市盈率',
|
||||
turnover_rate DECIMAL(10, 6) COMMENT '换手率(%)',
|
||||
volume BIGINT COMMENT '成交量(股)',
|
||||
turnover DECIMAL(20, 2) COMMENT '成交额(元)',
|
||||
change_rate DECIMAL(10, 6) COMMENT '涨跌幅(%)',
|
||||
last_close DECIMAL(10, 3) COMMENT '昨收价',
|
||||
market VARCHAR(10) COMMENT '市场标识',
|
||||
create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
|
||||
UNIQUE KEY uk_code_date (stock_code, trade_date),
|
||||
KEY idx_trade_date (trade_date),
|
||||
KEY idx_stock_code (stock_code)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='股票行情数据表'
|
||||
"""
|
||||
db.execute_update(create_table_sql)
|
||||
logging.info(f"创建了新表: {table_name}")
|
||||
|
||||
affected_rows = db.execute_many(insert_sql, processed_data)
|
||||
logging.info(f"成功插入/更新 {affected_rows} 条行情记录到表 {table_name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logging.error(f"保存行情数据到表 {table_name} 失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def read_missing_codes_basic(file_path='missing_tables.txt'):
|
||||
"""基础读取方法 - 按行读取所有内容"""
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
lines = f.readlines()
|
||||
# 去除每行末尾的换行符，并过滤空行
|
||||
codes = [line.strip() for line in lines if line.strip()]
|
||||
return codes
|
||||
except FileNotFoundError:
|
||||
print(f"文件 {file_path} 不存在")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f"读取文件失败: {str(e)}")
|
||||
return []
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('Debug.log', encoding='utf-8'), # 关键在这里
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
|
||||
# 数据库配置
|
||||
db_config = {
|
||||
'host': 'localhost',
|
||||
'user': 'root',
|
||||
'password': 'bzskmysql',
|
||||
'database': 'klinedata_1d_hk'
|
||||
}
|
||||
|
||||
|
||||
# market_data = get_market_data(Market.HK) # 获取香港市场数据,后面需要改成按照筛选来获取
|
||||
|
||||
market_data = read_missing_codes_basic()
|
||||
|
||||
nCount = 0 # 记录账号获取多少只股票的数据
|
||||
for code in market_data:
|
||||
|
||||
# nCount = nCount + 1
|
||||
# if nCount < 450:
|
||||
# continue
|
||||
|
||||
quote_ctx = OpenQuoteContext(host='127.0.0.1', port=11111)
|
||||
# ret, data, page_req_key = quote_ctx.request_history_kline(code, start='2024-09-01', end='2025-08-20', max_count=100, session=Session.ALL) # 每页5个，请求第一页 session 字段紧美股有用
|
||||
ret, data, page_req_key = quote_ctx.request_history_kline(code, start='2024-10-01', end='2025-08-20', max_count=100) # 每页5个，请求第一页
|
||||
# 保存数据到自定义表
|
||||
custom_table_name = 'hk_' + code[3:] # 自定义表名
|
||||
|
||||
if ret == RET_OK:
|
||||
success = save_quotes_to_db(db_config, data, table_name=custom_table_name)
|
||||
else:
|
||||
print('error:', data)
|
||||
|
||||
while page_req_key != None: # 请求后面的所有结果
|
||||
# print('*************************************')
|
||||
ret, data, page_req_key = quote_ctx.request_history_kline(code, start='2024-10-01', end='2025-08-20', max_count=100, page_req_key=page_req_key) # 请求翻页后的数据
|
||||
if ret == RET_OK:
|
||||
success = save_quotes_to_db(db_config, data, table_name=custom_table_name)
|
||||
else:
|
||||
print('error:', data)
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
# if nCount == 2000:
|
||||
# break
|
||||
quote_ctx.close() # 结束后记得关闭当条连接，防止连接条数用尽
|
||||
|
||||
|
||||
Reference in New Issue
Block a user