From 786c669d62eff2c7667a793c57ae1b0b650a9124 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 20 Aug 2025 14:09:35 +0800 Subject: [PATCH] updata updatekline.py --- .gitignore | 1 + PyCode/CalculateCapitalization_akshare.py | 298 -------- PyCode/ConditionalSelection.py | 31 +- PyCode/ExportData_akshare.py | 315 -------- PyCode/LogHelper.py | 89 +-- PyCode/MySQLHelper.py | 118 ++- PyCode/checktable.py | 55 +- PyCode/updatekline.py | 88 ++- PyCode/updatekline_akshare.py | 197 ----- config/HK_futu.txt | 892 ++++++++++++++++++++++ config/hang_futu.txt | 4 +- config/备注.txt | 6 +- 12 files changed, 1137 insertions(+), 957 deletions(-) delete mode 100644 PyCode/CalculateCapitalization_akshare.py delete mode 100644 PyCode/ExportData_akshare.py delete mode 100644 PyCode/updatekline_akshare.py create mode 100644 config/HK_futu.txt diff --git a/.gitignore b/.gitignore index 37274b2..23c5087 100644 --- a/.gitignore +++ b/.gitignore @@ -266,4 +266,5 @@ compile_commands.json # data file data/ +logs/ diff --git a/PyCode/CalculateCapitalization_akshare.py b/PyCode/CalculateCapitalization_akshare.py deleted file mode 100644 index 381b85b..0000000 --- a/PyCode/CalculateCapitalization_akshare.py +++ /dev/null @@ -1,298 +0,0 @@ -""" - 计算平均市值 - - 根据 conditionalselection 表格中的股票代码，查找日K数据表格 - 每个月计算一次结果存放在 hk_monthly_avg_2410_2508 表格中 - -""" - -import pandas as pd -from datetime import datetime -import logging -from futu import * -from pymysql import Error -from MySQLHelper import MySQLHelper # MySQLHelper类保存为单独文件 -from typing import Optional, List, Dict, Union, Tuple - - -def create_monthly_avg_table(db_config: dict, target_table: str = "monthly_close_avg") -> bool: - """ - 创建专门存储2024年10月至2025年8月月度均值的表结构 - - Args: - db_config: 数据库配置 - target_table: 目标表名 - - Returns: - bool: 是否创建成功 - """ - try: - with MySQLHelper(**db_config) as db: - create_sql = f""" - CREATE TABLE IF NOT EXISTS {target_table} ( - id INT AUTO_INCREMENT PRIMARY KEY, - stock_code VARCHAR(20) NOT NULL COMMENT '股票代码', - stock_name VARCHAR(50) COMMENT '股票名称', - ym_2410 DECIMAL(10, 3) COMMENT '2024年10月均收盘价', - ym_2411 DECIMAL(10, 3) COMMENT '2024年11月均收盘价', - ym_2412 DECIMAL(10, 3) COMMENT '2024年12月均收盘价', - ym_2501 DECIMAL(10, 3) COMMENT '2025年1月均收盘价', - ym_2502 DECIMAL(10, 3) COMMENT '2025年2月均收盘价', - ym_2503 DECIMAL(10, 3) COMMENT '2025年3月均收盘价', - ym_2504 DECIMAL(10, 3) COMMENT '2025年4月均收盘价', - ym_2505 DECIMAL(10, 3) COMMENT '2025年5月均收盘价', - ym_2506 DECIMAL(10, 3) COMMENT '2025年6月均收盘价', - ym_2507 DECIMAL(10, 3) COMMENT '2025年7月均收盘价', - ym_2508 DECIMAL(10, 3) COMMENT '2025年8月均收盘价', - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - UNIQUE KEY uk_stock_code (stock_code) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='月度收盘价均值表(2024.10-2025.08)' - """ - db.execute_update(create_sql) - # logging.info(f"创建/确认表 {target_table} 结构成功") - return True - except Exception as e: - logging.error(f"创建表失败: {str(e)}") - return False - -def calculate_and_save_monthly_avg(db_config: dict, - source_table: str = "stock_quotes", - target_table: str = "monthly_close_avg") -> bool: - """ - 计算并保存2024年10月至2025年8月的月度收盘价均值 - - Args: - db_config: 数据库配置 - source_table: 源数据表名 - target_table: 目标表名 - - Returns: - bool: 是否成功 - """ - # 定义分析的时间范围 - month_ranges = { - 'ym_2410': ('2024-10-01', '2024-10-31'), - 'ym_2411': ('2024-11-01', '2024-11-30'), - 'ym_2412': ('2024-12-01', '2024-12-31'), - 'ym_2501': ('2025-01-01', '2025-01-31'), - 'ym_2502': ('2025-02-01', '2025-02-28'), - 'ym_2503': ('2025-03-01', '2025-03-31'), - 'ym_2504': ('2025-04-01', '2025-04-30'), - 'ym_2505': ('2025-05-01', '2025-05-31'), - 'ym_2506': ('2025-06-01', '2025-06-30'), - 'ym_2507': ('2025-07-01', '2025-07-31'), - 'ym_2508': ('2025-08-01', '2025-08-31') - } - - try: - # 确保表结构存在 - if not create_monthly_avg_table(db_config, target_table): - return False - - with MySQLHelper(**db_config) as db: - # 获取所有股票代码和名称 - stock_info = db.execute_query( - f"SELECT DISTINCT stock_code, stock_name FROM {source_table}" - ) - - if not stock_info: - logging.error("没有获取到股票基本信息") - return False - - # 为每只股票计算各月均值 - for stock in stock_info: - stock_code = stock['stock_code'] - stock_name = stock['stock_name'] - - monthly_data = {'stock_code': stock_code, 'stock_name': stock_name} - - # 计算每个月的均值 - for month_col, (start_date, end_date) in month_ranges.items(): - sql = f""" - SELECT AVG(close_price) as avg_close - FROM {source_table} - WHERE stock_code = %s - AND trade_date BETWEEN %s AND %s - """ - result = db.execute_query(sql, (stock_code, start_date, end_date)) - monthly_data[month_col] = float(result[0]['avg_close']) if result and result[0]['avg_close'] else None - - # 插入或更新数据 - upsert_sql = f""" - INSERT INTO {target_table} ( - stock_code, stock_name, - ym_2410, ym_2411, ym_2412, - ym_2501, ym_2502, ym_2503, ym_2504, - ym_2505, ym_2506, ym_2507, ym_2508 - ) VALUES ( - %(stock_code)s, %(stock_name)s, - %(ym_2410)s, %(ym_2411)s, %(ym_2412)s, - %(ym_2501)s, %(ym_2502)s, %(ym_2503)s, %(ym_2504)s, - %(ym_2505)s, %(ym_2506)s, %(ym_2507)s, %(ym_2508)s - ) - ON DUPLICATE KEY UPDATE - stock_name = VALUES(stock_name), - ym_2410 = VALUES(ym_2410), - ym_2411 = VALUES(ym_2411), - ym_2412 = VALUES(ym_2412), - ym_2501 = VALUES(ym_2501), - ym_2502 = VALUES(ym_2502), - ym_2503 = VALUES(ym_2503), - ym_2504 = VALUES(ym_2504), - ym_2505 = VALUES(ym_2505), - ym_2506 = VALUES(ym_2506), - ym_2507 = VALUES(ym_2507), - ym_2508 = VALUES(ym_2508), - update_time = CURRENT_TIMESTAMP - """ - db.execute_update(upsert_sql, monthly_data) - - logging.info("月度均值计算和保存完成") - return True - except Exception as e: - logging.error(f"计算和保存月度均值失败: {str(e)}") - return False - -# 安全转换函数 -def safe_float(v) -> Optional[float]: - """安全转换为float，处理N/A和空值""" - try: - return float(v) if pd.notna(v) and str(v).upper() != 'N/A' else None - except (ValueError, TypeError): - return None - -def safe_int(v) -> Optional[int]: - """安全转换为int，处理N/A和空值""" - try: - return int(v) if pd.notna(v) and str(v).upper() != 'N/A' else None - except (ValueError, TypeError): - return None - -def safe_parse_date(date_str, date_format='%Y-%m-%d'): - """ - 安全解析日期字符串 - :param date_str: 日期字符串 - :param date_format: 日期格式 - :return: 解析后的datetime对象或None - """ - if not date_str or pd.isna(date_str) or str(date_str).strip() == '': - return None - try: - return datetime.strptime(str(date_str), date_format) - except ValueError: - logging.warning(f"无法解析日期字符串: {date_str}") - return None - -def validate_market_data(dataset: list) -> list: - """ - 验证市场数据有效性 - - Args: - dataset (list): 原始数据集 - - Returns: - list: 通过验证的数据集 - """ - validated_data = [] - for item in dataset: - try: - # 必要字段检查 - if not item.get('code') or not item.get('name'): - logging.warning(f"跳过无效数据: 缺少必要字段 code或name") - continue - - # 筛选股票名称 - if item.get('name')[-1] == 'R': - continue - - # 数值范围验证 - if item.get('lot_size') is not None and item['lot_size'] < 0: - logging.warning(f"股票 {item['code']} 的lot_size为负值: {item['lot_size']}") - item['lot_size'] = None - - validated_data.append(item) - except Exception as e: - logging.warning(f"数据验证失败，跳过记录 {item.get('code')}: {str(e)}") - continue - - return validated_data - -def get_market_data(market: Market) -> List[str]: - """ - 从Futu API获取指定市场的股票代码列表 - - Args: - market (Market): 市场枚举值，如 Market.SH, Market.SZ - - Returns: - List[str]: 股票代码列表 - """ - quote_ctx = OpenQuoteContext(host='127.0.0.1', port=11111) - try: - ret, data = quote_ctx.get_stock_basicinfo(market, SecurityType.STOCK) - if ret == RET_OK: - # 提取code列并转换为列表 - codes = data['code'].astype(str).tolist() - logging.info(f"获取到 {market} 市场 {len(codes)} 个股票代码") - return codes - else: - logging.error(f"获取股票代码失败: {data}") - return [] - except Exception as e: - logging.error(f"获取股票代码时发生异常: {str(e)}") - return [] - finally: - quote_ctx.close() - -def get_stock_codes() -> List[str]: - """从conditionalselection表获取所有股票代码""" - try: - with MySQLHelper(**db_config) as db: - sql = f"SELECT DISTINCT stock_code,stock_name FROM conditionalselection" - results = db.execute_query(sql) - return [ - row['stock_code'] - for row in results - if row['stock_code'] and (row.get('stock_name', '') and row['stock_name'][-1] != 'R') # 排除 name 以 R 结尾的票 - ] - except Exception as e: - logging.error(f"获取股票代码失败: {str(e)}") - return [] - -if __name__ == "__main__": - - # 配置日志 - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler('Debug.log', encoding='utf-8'), # 关键在这里 - logging.StreamHandler() - ] - ) - - # 数据库配置 - db_config = { - 'host': 'localhost', - 'user': 'root', - 'password': 'bzskmysql', - 'database': 'klinedata_1d_hk' - } - - # market_data = get_market_data(Market.HK) - market_data = get_stock_codes() # 使用按照价格和流通股数量筛选的那个表格 - - for code in market_data: - - tablename = 'hk_' + code[3:] - # 计算并保存月度均值 - success = calculate_and_save_monthly_avg( - db_config=db_config, - source_table=tablename, - target_table="hk_monthly_avg_2410_2508" - ) - - if success: - logging.info("月度均值处理成功完成") - else: - logging.error("处理过程中出现错误") \ No newline at end of file diff --git a/PyCode/ConditionalSelection.py b/PyCode/ConditionalSelection.py index b2aa4fc..5413b8a 100644 --- a/PyCode/ConditionalSelection.py +++ b/PyCode/ConditionalSelection.py @@ -1,16 +1,17 @@ +""" # -# 使用筛选器来更新股票列表 +# 使用筛选器来更新股票列表 <临时使用，功能是完整的，后面继续优化> # +""" from futu import * import time -import logging from typing import List, Dict from MySQLHelper import MySQLHelper # 假设您已有MySQLHelper类 from LogHelper import LogHelper # 基本用法（自动创建日期日志+控制台输出） -logger = LogHelper().setup() +logger = LogHelper(logger_name = 'floatShare').setup() class FutuStockFilter: def __init__(self, db_config:dict): @@ -24,7 +25,7 @@ class FutuStockFilter: self.quote_ctx = OpenQuoteContext(host='127.0.0.1', port=11111) return True except Exception as e: - logging.error(f"连接Futu API失败: {str(e)}") + logger.error(f"连接Futu API失败: {str(e)}") return False def disconnect(self): @@ -48,10 +49,10 @@ class FutuStockFilter: try: with MySQLHelper(**self.db_config) as db: db.execute_update(create_table_sql) - logging.info("数据库表准备就绪") + logger.info("数据库表准备就绪") return True except Exception as e: - logging.error(f"准备数据库表失败: {str(e)}") + logger.error(f"准备数据库表失败: {str(e)}") return False def process_filter_result(self, ret_list: List) -> List[Dict]: @@ -66,7 +67,7 @@ class FutuStockFilter: } processed_data.append(data) except Exception as e: - logging.error(f"处理股票 {getattr(item, 'stock_code', '未知')} 数据失败: {str(e)}") + logger.error(f"处理股票 {getattr(item, 'stock_code', '未知')} 数据失败: {str(e)}") return processed_data @@ -98,10 +99,10 @@ class FutuStockFilter: params.append(tuple(item.get(col) for col in columns)) affected_rows = db.execute_many(sql, params) - logging.info(f"批量保存了 {len(data_batch)} 条记录，影响 {affected_rows} 行") + logger.info(f"批量保存了 {len(data_batch)} 条记录，影响 {affected_rows} 行") return affected_rows except Exception as e: - logging.error(f"批量保存失败: {str(e)}") + logger.error(f"批量保存失败: {str(e)}") return 0 def run_direct_import(self): @@ -113,7 +114,7 @@ class FutuStockFilter: self.disconnect() return False - logging.info(f"================= start ===================") + logger.info(f"================= start ===================") simple_filter = SimpleFilter() simple_filter.filter_min = 0 simple_filter.filter_max = 100000000000000 @@ -134,7 +135,7 @@ class FutuStockFilter: if ret == RET_OK: last_page, all_count, ret_list = ls - logging.info(f'获取股票列表进度: {nBegin}/{all_count}') + logger.info(f'获取股票列表进度: {nBegin}/{all_count}') # 处理并保存当前批次数据 processed_data = self.process_filter_result(ret_list) @@ -144,17 +145,17 @@ class FutuStockFilter: nBegin += len(ret_list) else: - logging.error(f'获取股票列表错误: {ls}') + logger.error(f'获取股票列表错误: {ls}') break time.sleep(3) # 避免触发限频 - logging.info(f"导入完成! 共导入 {total_imported} 条记录") - logging.info(f"================= end ===================") + logger.info(f"导入完成! 共导入 {total_imported} 条记录") + logger.info(f"================= end ===================") return total_imported > 0 except Exception as e: - logging.error(f"导入过程中发生错误: {str(e)}") + logger.error(f"导入过程中发生错误: {str(e)}") return False finally: self.disconnect() \ No newline at end of file diff --git a/PyCode/ExportData_akshare.py b/PyCode/ExportData_akshare.py deleted file mode 100644 index 7a36bb8..0000000 --- a/PyCode/ExportData_akshare.py +++ /dev/null @@ -1,315 +0,0 @@ -import csv -import pandas as pd -from MySQLHelper import MySQLHelper -import logging -from typing import List, Dict, Optional, Tuple -from datetime import datetime - -def get_monthly_avg_data(db_config: dict, table_name: str) -> Optional[List[Dict]]: - """ - 从数据库读取月度均值数据 - - Args: - db_config: 数据库配置 - table_name: 源数据表名 - - Returns: - List[Dict]: 查询结果数据集，失败返回None - """ - try: - with MySQLHelper(**db_config) as db: - # 获取表结构信息 - columns = db.execute_query(f""" - SELECT COLUMN_NAME - FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s - ORDER BY ORDINAL_POSITION - """, (db_config['database'], table_name)) - - if not columns: - logging.error(f"表 {table_name} 不存在或没有列") - return None - - # 获取列名列表(排除id和update_time) - field_names = [col['COLUMN_NAME'] for col in columns - if col['COLUMN_NAME'] not in ('id', 'update_time')] - - # 查询数据 - data = db.execute_query(f""" - SELECT {', '.join(field_names)} - FROM {table_name} - ORDER BY stock_code - """) - - if not data: - logging.error(f"表 {table_name} 中没有数据") - return None - - return data - - except Exception as e: - logging.error(f"从数据库读取月度均值数据失败: {str(e)}") - return None - -def get_float_share_data(db_config: dict, table_name: str) -> Optional[List[Dict]]: - """ - 从conditionalselection表读取流通股本数据 - - Args: - db_config: 数据库配置 - table_name: 源数据表名 - - Returns: - List[Dict]: 查询结果数据集，失败返回None - """ - try: - with MySQLHelper(**db_config) as db: - # 查询流通股本数据 - data = db.execute_query(f""" - SELECT stock_code, stock_name, float_share - FROM {table_name} - ORDER BY stock_code - """) - - if not data: - logging.error(f"表 {table_name} 中没有流通股本数据") - return None - - return data - - except Exception as e: - logging.error(f"从数据库读取流通股本数据失败: {str(e)}") - return None - -def merge_data(monthly_data: List[Dict], float_share_data: List[Dict]) -> List[Dict]: - """ - 合并月度均价数据和流通股本数据 - - Args: - monthly_data: 月度均价数据 - float_share_data: 流通股本数据 - - Returns: - List[Dict]: 合并后的数据集 - """ - merged_data = [] - float_share_dict = {item['stock_code']: item['float_share'] for item in float_share_data} - - for item in monthly_data: - merged_item = item.copy() - merged_item['float_share'] = float_share_dict.get(item['stock_code'], 'N/A') - merged_data.append(merged_item) - - return merged_data - -def export_to_csv(data: List[Dict], output_file: str) -> bool: - """ - 将合并后的数据导出到CSV文件 - - Args: - data: 要导出的数据集 - output_file: 输出的CSV文件路径 - - Returns: - bool: 是否导出成功 - """ - if not data: - return False - - try: - # 获取字段名(使用第一个数据的键) - field_names = list(data[0].keys()) - - # 字段名到中文的映射 - header_map = { - 'stock_code': '股票代码', - 'stock_name': '股票名称', - 'float_share': '流通股本(千股)', - 'ym_2410': '2024年10月均收盘价', - 'ym_2411': '2024年11月均收盘价', - 'ym_2412': '2024年12月均收盘价', - 'ym_2501': '2025年1月均收盘价', - 'ym_2502': '2025年2月均收盘价', - 'ym_2503': '2025年3月均收盘价', - 'ym_2504': '2025年4月均收盘价', - 'ym_2505': '2025年5月均收盘价', - 'ym_2506': '2025年6月均收盘价', - 'ym_2507': '2025年7月均收盘价', - 'ym_2508': '2025年8月均收盘价' - } - - with open(output_file, mode='w', newline='', encoding='utf-8-sig') as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=field_names) - - # 写入中文表头 - writer.writerow({col: header_map.get(col, col) for col in field_names}) - - # 写入数据 - writer.writerows(data) - - logging.info(f"成功导出 {len(data)} 条记录到CSV文件: {output_file}") - return True - - except Exception as e: - logging.error(f"导出到CSV失败: {str(e)}") - return False - -def export_to_excel(data: List[Dict], output_file: str) -> bool: - """ - 将合并后的数据导出为Excel文件(包含多个工作表) - - Args: - data: 要导出的数据集 - output_file: 输出的Excel文件路径 - - Returns: - bool: 是否导出成功 - """ - if not data: - return False - - try: - # 转换为DataFrame - df = pd.DataFrame(data) - - # 设置股票代码为索引 - if 'stock_code' in df.columns: - df.set_index('stock_code', inplace=True) - - # 创建Excel writer对象 - with pd.ExcelWriter(output_file, engine='openpyxl') as writer: - # 1. 原始数据工作表 - df.to_excel(writer, sheet_name='合并数据') - - # 2. 统计信息工作表（仅当有数值列时） - numeric_cols = [col for col in df.columns if col.startswith('ym_') and pd.api.types.is_numeric_dtype(df[col])] - - if numeric_cols: - try: - stats = df[numeric_cols].describe().loc[['mean', 'min', 'max', 'std']] - stats.to_excel(writer, sheet_name='统计信息') - except KeyError: - logging.warning("无法生成完整的统计信息，数据可能不足") - # 生成简化版统计信息 - stats = df[numeric_cols].agg(['mean', 'min', 'max', 'std']) - stats.to_excel(writer, sheet_name='统计信息') - - # 3. 涨幅排名工作表（需要至少两个月份数据） - if len(numeric_cols) >= 2: - first_month = numeric_cols[0] - last_month = numeric_cols[-1] - - try: - df['涨幅(%)'] = (df[last_month] - df[first_month]) / df[first_month] * 100 - result_df = df[['stock_name', '涨幅(%)', 'float_share']].copy() - result_df.dropna(subset=['涨幅(%)'], inplace=True) - result_df.sort_values('涨幅(%)', ascending=False, inplace=True) - result_df.to_excel(writer, sheet_name='涨幅排名') - except Exception as e: - logging.warning(f"无法计算涨幅: {str(e)}") - - # 4. 月度趋势工作表 - if numeric_cols: - try: - trend_df = df[numeric_cols].transpose() - trend_df.index = [col.replace('ym_', '') for col in numeric_cols] - trend_df.to_excel(writer, sheet_name='月度趋势') - except Exception as e: - logging.warning(f"无法生成月度趋势: {str(e)}") - - # 5. 流通股本分析工作表 - if 'float_share' in df.columns and pd.api.types.is_numeric_dtype(df['float_share']): - try: - float_stats = df['float_share'].describe().to_frame().T - float_stats.to_excel(writer, sheet_name='流通股本分析') - except Exception as e: - logging.warning(f"无法生成流通股本分析: {str(e)}") - - logging.info(f"成功导出Excel文件: {output_file}") - return True - - except Exception as e: - logging.error(f"导出Excel失败: {str(e)}") - return False - -def export_combined_data(db_config: dict, - monthly_table: str, - float_share_table: str, - csv_file: str = None, - excel_file: str = None) -> bool: - """ - 导出合并后的数据到CSV和/或Excel - - Args: - db_config: 数据库配置 - monthly_table: 月度均价表名 - float_share_table: 流通股本表名 - csv_file: CSV输出路径(可选) - excel_file: Excel输出路径(可选) - - Returns: - bool: 是否至少有一种格式导出成功 - """ - if not csv_file and not excel_file: - logging.error("必须指定至少一种输出格式") - return False - - # 从数据库获取数据 - monthly_data = get_monthly_avg_data(db_config, monthly_table) - if not monthly_data: - logging.error("无法获取月度均价数据") - return False - - float_share_data = get_float_share_data(db_config, float_share_table) - if not float_share_data: - logging.error("无法获取流通股本数据") - return False - - # 合并数据 - merged_data = merge_data(monthly_data, float_share_data) - - # 导出结果 - csv_success = True - excel_success = True - - if csv_file: - csv_success = export_to_csv(merged_data, csv_file) - - if excel_file: - excel_success = export_to_excel(merged_data, excel_file) - - return csv_success or excel_success - -# 主程序入口 -if __name__ == "__main__": - # 配置日志 - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler('export_combined_data.log', encoding='utf-8'), - logging.StreamHandler() - ] - ) - - # 数据库配置 - db_config = { - 'host': 'localhost', - 'user': 'root', - 'password': 'bzskmysql', - 'database': 'klinedata_1d_hk_akshare' - } - - # 导出合并数据 - success = export_combined_data( - db_config=db_config, - monthly_table="hk_monthly_avg_2410_2508", - float_share_table="conditionalselection", - csv_file="hk_stocks_combined_data.csv", - excel_file="hk_stocks_combined_data.xlsx" - ) - - if success: - logging.info("数据合并导出成功完成") - else: - logging.error("数据合并导出过程中出现错误") diff --git a/PyCode/LogHelper.py b/PyCode/LogHelper.py index 7038257..1de09be 100644 --- a/PyCode/LogHelper.py +++ b/PyCode/LogHelper.py @@ -1,47 +1,9 @@ -# import logging -# import sys - -# class LogHelper: -# def __init__(self, -# level=logging.INFO, -# format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', -# handlers=None): -# """ -# 初始化日志配置 - -# :param level: 日志级别，默认为 logging.INFO -# :param format: 日志格式字符串 -# :param handlers: 日志处理器列表，默认为空（会自动添加控制台处理器） -# """ -# self.level = level -# self.format = format -# self.handlers = handlers if handlers is not None else [] - -# def add_console_handler(self, stream=sys.stdout, encoding='utf-8'): -# """添加控制台处理器""" -# console_handler = logging.StreamHandler(stream) -# console_handler.setFormatter(logging.Formatter(self.format)) -# # 设置控制台编码 -# if encoding: -# console_handler.encoding = encoding -# self.handlers.append(console_handler) - -# def add_file_handler(self, filename, encoding='utf-8'): -# """添加文件处理器（解决中文乱码问题）""" -# # 使用支持UTF-8编码的文件处理器 -# filepath = "log/" + filename -# file_handler = logging.FileHandler(filepath, encoding=encoding) -# file_handler.setFormatter(logging.Formatter(self.format)) -# self.handlers.append(file_handler) - -# def setup(self): -# """应用日志配置""" -# logging.basicConfig( -# level=self.level, -# format=self.format, -# handlers=self.handlers -# ) +""" +日志管理类 +日志输出到运行目录指定文件夹 +日志文件按照：YYYY-MM-dd.log 格式输出 +""" import logging import sys @@ -49,11 +11,21 @@ import os from datetime import datetime class LogHelper: + # 内部日志级别映射表 + _LEVEL_MAP = { + 'DEBUG': 10, + 'INFO': 20, + 'WARNING': 30, + 'ERROR': 40, + 'CRITICAL': 50 + } + def __init__(self, - level=logging.INFO, + level='INFO', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=None, - log_dir="logs"): + log_dir="logs", + logger_name=None): """ 初始化日志配置 @@ -62,10 +34,11 @@ class LogHelper: :param handlers: 日志处理器列表，默认为空（会自动添加控制台处理器） :param log_dir: 日志存储目录，默认为"logs" """ - self.level = level + self.level = self._LEVEL_MAP.get(level.upper(), 20) # 默认INFO self.format = format self.handlers = handlers if handlers is not None else [] self.log_dir = log_dir + self.logger_name = logger_name # 确保日志目录存在 os.makedirs(self.log_dir, exist_ok=True) @@ -76,6 +49,7 @@ class LogHelper: if encoding: console_handler.encoding = encoding self.handlers.append(console_handler) + return self # 支持链式调用 def _get_daily_log_path(self): """生成基于当前日期的日志文件路径""" @@ -89,6 +63,7 @@ class LogHelper: file_handler = logging.FileHandler(log_path, encoding=encoding) file_handler.setFormatter(logging.Formatter(self.format)) self.handlers.append(file_handler) + return self # 支持链式调用 def setup(self): """应用日志配置（自动添加日期文件处理器）""" @@ -97,9 +72,19 @@ class LogHelper: self.add_console_handler() self._add_daily_file_handler() - # 应用配置 - logging.basicConfig( - level=self.level, - format=self.format, - handlers=self.handlers - ) \ No newline at end of file + # 获取或创建logger + logger = logging.getLogger(self.logger_name) + logger.setLevel(self.level) + + # 移除所有现有处理器（避免重复添加） + for handler in logger.handlers[:]: + logger.removeHandler(handler) + + # 添加配置的处理器 + for handler in self.handlers: + logger.addHandler(handler) + + # 确保日志消息不会传递给父logger（避免重复记录） + logger.propagate = False + + return logger \ No newline at end of file diff --git a/PyCode/MySQLHelper.py b/PyCode/MySQLHelper.py index 8055872..23d7288 100644 --- a/PyCode/MySQLHelper.py +++ b/PyCode/MySQLHelper.py @@ -1,6 +1,25 @@ +""" + MySqlHelper 增强版 + —— 增加事务管理 + —— 增加ID获取 + —— 增加表操作等使用功能 +""" import pymysql from pymysql import Error from typing import List, Dict, Union, Optional, Tuple +from contextlib import contextmanager +from LogHelper import LogHelper + +# 基本用法（自动创建日期日志+控制台输出） +logger = LogHelper(logger_name = 'database').setup() + +# # 高级用法（自定义配置） +# logger = LogHelper( +# level=logging.DEBUG, +# log_dir="databaselogs", +# format='%(levelname)s - %(message)s' +# ).setup() + class MySQLHelper: def __init__(self, host: str, user: str, password: str, database: str, @@ -39,10 +58,10 @@ class MySQLHelper: cursorclass=pymysql.cursors.DictCursor # 返回字典形式的结果 ) self.cursor = self.connection.cursor() - print("MySQL数据库连接成功") + logger.info("MySQL数据库连接成功") return True except Error as e: - print(f"连接MySQL数据库失败: {e}") + logger.error(f"连接MySQL数据库失败: {e}") return False def close(self) -> None: @@ -53,7 +72,7 @@ class MySQLHelper: self.cursor.close() if self.connection: self.connection.close() - print("MySQL数据库连接已关闭") + logger.info("MySQL数据库连接已关闭") def execute_query(self, sql: str, params: Union[Tuple, List, Dict, None] = None) -> List[Dict]: """ @@ -66,7 +85,7 @@ class MySQLHelper: self.cursor.execute(sql, params) return self.cursor.fetchall() except Error as e: - print(f"查询执行失败: {e}") + logger.error(f"查询执行失败: {e}") return [] def execute_update(self, sql: str, params: Union[Tuple, List, Dict, None] = None) -> int: @@ -82,7 +101,7 @@ class MySQLHelper: return affected_rows except Error as e: self.connection.rollback() - print(f"更新执行失败: {e}") + logger.error(f"更新执行失败: {e}") return 0 def execute_many(self, sql: str, params_list: List[Union[Tuple, List, Dict]]) -> int: @@ -98,9 +117,41 @@ class MySQLHelper: return affected_rows except Error as e: self.connection.rollback() - print(f"批量执行失败: {e}") + logger.error(f"批量执行失败: {e}") return 0 + # ================== 新增功能方法 ================== + def get_last_insert_id(self) -> int: + """ + 获取最后插入行的自增ID + :return: 自增ID值 + """ + return self.cursor.lastrowid if self.cursor else 0 + + def execute_insert(self, sql: str, params: Union[Tuple, List, Dict, None] = None) -> int: + """ + 执行插入操作并返回自增ID + :return: 自增ID值 + """ + self.execute_update(sql, params) + return self.get_last_insert_id() + + @contextmanager + def transaction(self): + """ + 事务上下文管理器，确保操作原子性 + 用法： + with db.transaction(): + db.execute_update(...) + db.execute_many(...) + """ + try: + yield + self.connection.commit() + except Exception as e: + self.connection.rollback() + raise e + def get_one(self, sql: str, params: Union[Tuple, List, Dict, None] = None) -> Optional[Dict]: """ 获取单条记录 @@ -112,7 +163,7 @@ class MySQLHelper: self.cursor.execute(sql, params) return self.cursor.fetchone() except Error as e: - print(f"获取单条记录失败: {e}") + logger.error(f"获取单条记录失败: {e}") return None def table_exists(self, table_name: str) -> bool: @@ -124,12 +175,61 @@ class MySQLHelper: sql = "SHOW TABLES LIKE %s" result = self.execute_query(sql, (table_name,)) return len(result) > 0 + + def create_table(self, sql: str) -> bool: + """ + 执行建表语句 + :param sql: CREATE TABLE语句 + :return: 是否成功 + """ + try: + self.cursor.execute(sql) + self.connection.commit() + return True + except Error as e: + logger.error(f"创建表失败: {e}") + return False + + def drop_table(self, table_name: str) -> bool: + """ + 删除表 + :param table_name: 表名 + :return: 是否成功 + """ + try: + self.cursor.execute(f"DROP TABLE IF EXISTS {table_name}") + self.connection.commit() + return True + except Error as e: + logger.error(f"删除表失败: {e}") + return False + def get_columns(self, table_name: str) -> List[Dict]: + """ + 获取表的列信息 + :param table_name: 表名 + :return: 列信息字典列表 + """ + return self.execute_query(f"DESCRIBE {table_name}") + + +# ================== 事务控制方法 ================== + def start_transaction(self): + """显式开始事务""" + self.connection.begin() + + def commit(self): + """提交事务""" + self.connection.commit() + + def rollback(self): + """回滚事务""" + self.connection.rollback() + + # ================== 上下文管理器 ================== def __enter__(self): - """支持with上下文管理""" self.connect() return self def __exit__(self, exc_type, exc_val, exc_tb): - """支持with上下文管理""" self.close() \ No newline at end of file diff --git a/PyCode/checktable.py b/PyCode/checktable.py index 148cd1a..dfe2a5a 100644 --- a/PyCode/checktable.py +++ b/PyCode/checktable.py @@ -1,34 +1,29 @@ """ 检查K线数据是否下载完成 + + 检测表格中的股票是否都有对应的表格 """ -import logging from typing import List from MySQLHelper import MySQLHelper # 假设您已有MySQLHelper类 +from LogHelper import LogHelper -# 配置日志 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler('Debug.log', encoding='utf-8'), # 关键在这里 - logging.StreamHandler() - ] -) +# 基本用法（自动创建日期日志+控制台输出） +logger = LogHelper(logger_name = 'checkTable').setup() class StockTableChecker: def __init__(self, db_config: dict): self.db_config = db_config - self.conditional_selection_table = "conditionalselection" + self.stock_list_table = "stock_filter" def get_stock_codes(self) -> List[str]: """从conditionalselection表获取所有股票代码""" try: with MySQLHelper(**self.db_config) as db: - sql = f"SELECT DISTINCT stock_code FROM {self.conditional_selection_table}" + sql = f"SELECT DISTINCT stock_code FROM {self.stock_list_table}" results = db.execute_query(sql) return [row['stock_code'] for row in results if row['stock_code']] except Exception as e: - logging.error(f"获取股票代码失败: {str(e)}") + logger.error(f"获取股票代码失败: {str(e)}") return [] def check_tables_exist(self, stock_codes: List[str]) -> dict: @@ -57,18 +52,18 @@ class StockTableChecker: return result except Exception as e: - logging.error(f"检查表存在性失败: {str(e)}") + logger.error(f"检查表存在性失败: {str(e)}") return {'exists': [], 'not_exists': []} - def write_missing_codes_to_txt(self, missing_codes: list, filename: str = "missing_tables.txt"): + def write_missing_codes_to_txt(self, missing_codes: list, filename: str = "data\missing_tables.txt"): """将缺失的股票代码写入TXT文件""" try: with open(filename, 'w', encoding='utf-8') as f: for code in missing_codes: f.write(f"{code}\n") - logging.info(f"已将 {len(missing_codes)} 个缺失表对应的股票代码写入 {filename}") + logger.info(f"已将 {len(missing_codes)} 个缺失表对应的股票代码写入 {filename}") except Exception as e: - logging.error(f"写入TXT文件失败: {str(e)}") + logger.error(f"写入TXT文件失败: {str(e)}") def read_missing_codes_basic(file_path='missing_tables.txt'): """基础读取方法 - 按行读取所有内容""" @@ -88,15 +83,15 @@ class StockTableChecker: def run_check(self): """执行完整的检查流程""" - logging.info("开始检查股票代码对应表...") + logger.info("开始检查股票代码对应表...") # 1. 获取所有股票代码 stock_codes = self.get_stock_codes() if not stock_codes: - logging.error("没有获取到任何股票代码") + logger.error("没有获取到任何股票代码") return - logging.info(f"共获取到 {len(stock_codes)} 个股票代码") + logger.info(f"共获取到 {len(stock_codes)} 个股票代码") # 2. 检查表存在性 check_result = self.check_tables_exist(stock_codes) @@ -104,20 +99,20 @@ class StockTableChecker: not_exists_count = len(check_result['not_exists']) # 3. 输出结果 - logging.info("\n检查结果:") - logging.info(f"存在的表数量: {exists_count}") - logging.info(f"不存在的表数量: {not_exists_count}") + logger.info("\n检查结果:") + logger.info(f"存在的表数量: {exists_count}") + logger.info(f"不存在的表数量: {not_exists_count}") if not_exists_count > 0: - logging.info("\n不存在的表对应的股票代码:") + logger.info("\n不存在的表对应的股票代码:") for code in check_result['not_exists']: - logging.info(code) + logger.info(code) # 4. 统计信息 - logging.info("\n统计摘要:") - logging.info(f"总股票代码数: {len(stock_codes)}") - logging.info(f"存在对应表的比例: {exists_count/len(stock_codes):.2%}") - logging.info(f"缺失对应表的比例: {not_exists_count/len(stock_codes):.2%}") + logger.info("\n统计摘要:") + logger.info(f"总股票代码数: {len(stock_codes)}") + logger.info(f"存在对应表的比例: {exists_count/len(stock_codes):.2%}") + logger.info(f"缺失对应表的比例: {not_exists_count/len(stock_codes):.2%}") # 4. 将缺失的股票代码写入TXT文件 if check_result['not_exists']: @@ -131,7 +126,7 @@ if __name__ == "__main__": 'host': 'localhost', 'user': 'root', 'password': 'bzskmysql', - 'database': 'klinedata_1d_hk' + 'database': 'hk_kline_1d' } # 创建检查器并运行 diff --git a/PyCode/updatekline.py b/PyCode/updatekline.py index 8917d5e..3ddab2d 100644 --- a/PyCode/updatekline.py +++ b/PyCode/updatekline.py @@ -10,18 +10,20 @@ —— 根据股票代码，获取股票历史数据 —— 根据股票代码，获取流通股数量 —— 根据股票代码，新建/更新数据表 + —— 更新完成需检查 ，数据是否完整 -> checktable.py 操作界面做好了之后，再融合到整体代码中 """ from futu import * -from pymysql import Error from MySQLHelper import MySQLHelper # MySQLHelper类保存为单独文件 from LogHelper import LogHelper from datetime import datetime -import logging -from typing import Optional, List, Dict, Union, Tuple -import time +from typing import Optional, List, Dict from ConditionalSelection import FutuStockFilter from tqdm import tqdm import pandas as pd +import time + +# 基本用法（自动创建日期日志+控制台输出） +logger = LogHelper(logger_name = 'KLine').setup() def get_market_data(market: Market) -> List[str]: """ @@ -39,13 +41,13 @@ def get_market_data(market: Market) -> List[str]: if ret == RET_OK: # 提取code列并转换为列表 codes = data['code'].astype(str).tolist() - logging.info(f"获取到 {market} 市场 {len(codes)} 个股票代码") + logger.info(f"获取到 {market} 市场 {len(codes)} 个股票代码") return codes else: - logging.error(f"获取股票代码失败: {data}") + logger.error(f"获取股票代码失败: {data}") return [] except Exception as e: - logging.error(f"获取股票代码时发生异常: {str(e)}") + logger.error(f"获取股票代码时发生异常: {str(e)}") return [] finally: quote_ctx.close() @@ -89,7 +91,7 @@ def preprocess_quote_data(df: pd.DataFrame, float_share: Optional[int] = None) - } processed_data.append(item) except Exception as e: - logging.warning(f"处理行情数据时跳过异常行 {row.get('code', '未知')}: {str(e)}") + logger.warning(f"处理行情数据时跳过异常行 {row.get('code', '未知')}: {str(e)}") continue return processed_data @@ -115,7 +117,7 @@ def get_float_share(quote_ctx: OpenQuoteContext, code: str) -> Optional[int]: return int(float_share) return None except Exception as e: - logging.error(f"获取股票 {code} 的流通股数量失败: {str(e)}") + logger.error(f"获取股票 {code} 的流通股数量失败: {str(e)}") return None def get_float_share_data(db_config: dict, table_name: str) -> Optional[Dict[str, int]]: @@ -139,7 +141,7 @@ def get_float_share_data(db_config: dict, table_name: str) -> Optional[Dict[str, """) if not data: - logging.error(f"表 {table_name} 中没有流通股本数据") + logger.error(f"表 {table_name} 中没有流通股本数据") return None # 构建股票代码到流通股数量的映射字典 @@ -150,11 +152,11 @@ def get_float_share_data(db_config: dict, table_name: str) -> Optional[Dict[str, if stock_code and float_share is not None: float_share_dict[stock_code] = int(float_share) - logging.info(f"成功从 {table_name} 表加载 {len(float_share_dict)} 条流通股数据") + logger.info(f"成功从 {table_name} 表加载 {len(float_share_dict)} 条流通股数据") return float_share_dict except Exception as e: - logging.error(f"从数据库读取流通股本数据失败: {str(e)}") + logger.error(f"从数据库读取流通股本数据失败: {str(e)}") return None def save_quotes_to_db(db_config: dict, quote_data: pd.DataFrame, table_name: str = 'stock_quotes', float_share: Optional[int] = None) -> bool: @@ -173,7 +175,7 @@ def save_quotes_to_db(db_config: dict, quote_data: pd.DataFrame, table_name: str # 预处理数据 processed_data = preprocess_quote_data(quote_data, float_share) if not processed_data: - logging.error("没有有效数据需要保存") + logger.error("没有有效数据需要保存") return False # 动态生成SQL插入语句 @@ -230,13 +232,13 @@ def save_quotes_to_db(db_config: dict, quote_data: pd.DataFrame, table_name: str ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='股票行情数据表' """ db.execute_update(create_table_sql) - logging.info(f"创建了新表: {table_name}") + logger.info(f"创建了新表: {table_name}") affected_rows = db.execute_many(insert_sql, processed_data) - logging.info(f"成功插入/更新 {affected_rows} 条行情记录到表 {table_name}") + logger.info(f"成功插入/更新 {affected_rows} 条行情记录到表 {table_name}") return True except Exception as e: - logging.error(f"保存行情数据到表 {table_name} 失败: {str(e)}") + logger.error(f"保存行情数据到表 {table_name} 失败: {str(e)}") return False def read_single_account_stock_codes(file_path='data\missing_tables_小航富途.txt'): @@ -262,22 +264,28 @@ def get_stock_codes() -> List[str]: results = db.execute_query(sql) return [row['stock_code'] for row in results if row['stock_code']] except Exception as e: - logging.error(f"获取股票代码失败: {str(e)}") + logger.error(f"获取股票代码失败: {str(e)}") return [] +def write_missing_codes_to_txt(missing_codes: list, filename: str = "config\HK_futu.txt"): + """将缺失的股票代码追加到TXT文件，如果文件不存在则创建""" + try: + # 确保目录存在 + directory = os.path.dirname(filename) + if directory and not os.path.exists(directory): + os.makedirs(directory) + logger.info(f"创建目录: {directory}") + + # # 检查文件是否存在 + # file_exists = os.path.exists(filename) + with open(filename, 'a', encoding='utf-8') as f: + for code in missing_codes: + f.write(f"\n{code}") + logger.info(f"已将 {len(missing_codes)} 个缺失表对应的股票代码写入 {filename}") + except Exception as e: + logger.error(f"写入TXT文件失败: {str(e)}") if __name__ == "__main__": - - # 基本用法（自动创建日期日志+控制台输出） - logger = LogHelper().setup() - - # # 高级用法（自定义配置） - # logger = LogHelper( - # level=logging.DEBUG, - # log_dir="my_logs", - # format='%(levelname)s - %(message)s' - # ).setup() - # 数据库配置 db_config = { 'host': 'localhost', @@ -286,19 +294,25 @@ if __name__ == "__main__": 'database': 'hk_kline_1d' } - # 创建导入器并运行, 用于每日更新流通股数量 + # 创建导入器并运行, 用于每日更新流通股数量 -> 操作界面中增加一个开关，每天只需要更新一次 futuStockFilter = FutuStockFilter(db_config) futuStockFilter.run_direct_import() - # 每个账号获取的数据独立开来 + # 每个账号获取的数据独立开来 -> 操作见面可以选择 market_data_all = get_stock_codes() market_data_hang = read_single_account_stock_codes('config\hang_futu.txt') market_data_kevin= read_single_account_stock_codes('config\kevin_futu.txt') - - market_data = list(set(market_data_all) - set(market_data_hang) - set(market_data_kevin)) + market_data_HK= read_single_account_stock_codes('config\HK_futu.txt') + market_data_new = list(set(market_data_all) - set(market_data_hang) - set(market_data_kevin) - set(market_data_HK)) - # 每天收盘后更新数据 - start_date = (datetime.now() - timedelta(days = 3 * 365)).strftime("%Y-%m-%d") + # write_missing_codes_to_txt(market_data_new) # 新股票添加到文件中 -> 暂时手动设置 + + # 动态调整 + """*********************************************""" + market_data = market_data_new + + # 每天收盘后更新数据 -> 操作界面中，这个参数需要放出来 + start_date = (datetime.now() - timedelta(days = 5)).strftime("%Y-%m-%d") end_date = (datetime.now() + timedelta(days = 1)).strftime("%Y-%m-%d") # 获取流通股数据字典 @@ -319,7 +333,7 @@ if __name__ == "__main__": if float_share is not None: logger.info(f"从API获取股票 {code} 的流通股数量: {float_share}") else: - logging.warning(f"无法获取股票 {code} 的流通股数量") + logger.warning(f"无法获取股票 {code} 的流通股数量") # 获取历史K线数据 ret, data, page_req_key = quote_ctx.request_history_kline(code, start=start_date, end=end_date, max_count=100) @@ -341,9 +355,9 @@ if __name__ == "__main__": else: logger.error(f'error:{ data}') - time.sleep(1) + time.sleep(0.7) if isWhile == False: - time.sleep(1) + time.sleep(0.7) quote_ctx.close() # 结束后记得关闭当条连接，防止连接条数用尽 diff --git a/PyCode/updatekline_akshare.py b/PyCode/updatekline_akshare.py deleted file mode 100644 index 13a0628..0000000 --- a/PyCode/updatekline_akshare.py +++ /dev/null @@ -1,197 +0,0 @@ -""" - 该代码用于更新日 K 数据 -""" -from futu import * -from pymysql import Error -from MySQLHelper import MySQLHelper # MySQLHelper类保存为单独文件 -from datetime import datetime -import logging -from typing import Optional, List, Dict, Union, Tuple -import time -import akshare as ak - -def get_market_data(market: Market) -> List[str]: - """ - 从Futu API获取指定市场的股票代码列表 - - Args: - market (Market): 市场枚举值，如 Market.SH, Market.SZ - - Returns: - List[str]: 股票代码列表 - """ - quote_ctx = OpenQuoteContext(host='127.0.0.1', port=11111) - try: - ret, data = quote_ctx.get_stock_basicinfo(market, SecurityType.STOCK) - if ret == RET_OK: - # 提取code列并转换为列表 - codes = data['code'].astype(str).tolist() - logging.info(f"获取到 {market} 市场 {len(codes)} 个股票代码") - return codes - else: - logging.error(f"获取股票代码失败: {data}") - return [] - except Exception as e: - logging.error(f"获取股票代码时发生异常: {str(e)}") - return [] - finally: - quote_ctx.close() - -def preprocess_quote_data(df: pd.DataFrame) -> List[Dict]: - """ - 预处理行情数据，转换为适合数据库存储的格式 - - Args: - df (pd.DataFrame): 原始行情数据DataFrame - - Returns: - List[Dict]: 处理后的数据列表 - """ - processed_data = [] - for _, row in df.iterrows(): - try: - # 提取市场标识 - # market = row['code'].split('.')[0] if '.' in row['code'] else 'UNKNOWN' - - # 转换时间格式 - # trade_time = datetime.strptime(row['date'], '%Y-%m-%d %H:%M:%S') - - item = { - 'trade_date': str(row['date']), - 'open_price': float(row['open']), - 'close_price': float(row['close']), - 'high_price': float(row['high']), - 'low_price': float(row['low']), - 'volume': int(row['volume']) if pd.notna(row['volume']) else None, - } - processed_data.append(item) - except Exception as e: - logging.warning(f"处理行情数据时跳过异常行 {row.get('code', '未知')}: {str(e)}") - continue - - return processed_data - -def save_quotes_to_db(db_config: dict, quote_data: pd.DataFrame, table_name: str = 'stock_quotes') -> bool: - """ - 将行情数据保存到数据库 - - Args: - db_config (dict): 数据库配置 - quote_data (pd.DataFrame): 行情数据DataFrame - table_name (str): 目标表名(默认为'stock_quotes') - - Returns: - bool: 是否成功保存 - """ - # 预处理数据 - processed_data = preprocess_quote_data(quote_data) - if not processed_data: - logging.error("没有有效数据需要保存") - return False - - # 动态生成SQL插入语句 - insert_sql = f""" - INSERT INTO {table_name} ( - trade_date, open_price, close_price, high_price, low_price, volume - ) VALUES ( - %(trade_date)s, %(open_price)s, %(close_price)s, %(high_price)s, %(low_price)s, %(volume)s - ) - ON DUPLICATE KEY UPDATE - open_price = VALUES(open_price), - close_price = VALUES(close_price), - high_price = VALUES(high_price), - low_price = VALUES(low_price), - volume = VALUES(volume) - """ - - try: - with MySQLHelper(**db_config) as db: - # 检查表是否存在，不存在则创建 - if not db.table_exists(table_name): - create_table_sql = f""" - CREATE TABLE IF NOT EXISTS {table_name} ( - id INT AUTO_INCREMENT PRIMARY KEY, - trade_date DATETIME NOT NULL COMMENT '交易日期时间', - open_price DECIMAL(10, 3) COMMENT '开盘价', - close_price DECIMAL(10, 3) COMMENT '收盘价', - high_price DECIMAL(10, 3) COMMENT '最高价', - low_price DECIMAL(10, 3) COMMENT '最低价', - volume BIGINT COMMENT '成交量(股)', - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - UNIQUE KEY idx_trade_date (trade_date) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='股票行情数据表' - """ - db.execute_update(create_table_sql) - logging.info(f"创建了新表: {table_name}") - - affected_rows = db.execute_many(insert_sql, processed_data) - logging.info(f"成功插入/更新 {affected_rows} 条行情记录到表 {table_name}") - return True - except Exception as e: - logging.error(f"保存行情数据到表 {table_name} 失败: {str(e)}") - return False - -def read_missing_codes_basic(file_path='data\missing_tables_小航富途.txt'): - """基础读取方法 - 按行读取所有内容""" - try: - with open(file_path, 'r', encoding='utf-8') as f: - lines = f.readlines() - # 去除每行末尾的换行符，并过滤空行 - codes = [line.strip() for line in lines if line.strip()] - return codes - except FileNotFoundError: - print(f"文件 {file_path} 不存在") - return [] - except Exception as e: - print(f"读取文件失败: {str(e)}") - return [] - -def get_stock_codes() -> List[str]: - """从conditionalselection表获取所有股票代码""" - try: - with MySQLHelper(**db_config) as db: - sql = f"SELECT DISTINCT stock_code FROM conditionalselection" - results = db.execute_query(sql) - return [row['stock_code'] for row in results if row['stock_code']] - except Exception as e: - logging.error(f"获取股票代码失败: {str(e)}") - return [] - -if __name__ == "__main__": - # 配置日志 - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler('Debug.log', encoding='utf-8'), # 关键在这里 - logging.StreamHandler() - ] - ) - - # 数据库配置 - db_config = { - 'host': 'localhost', - 'user': 'root', - 'password': 'bzskmysql', - 'database': 'klinedata_1d_hk_akshare' - } - - - # market_data = get_market_data(Market.HK) # 获取香港市场数据,后面需要改成按照筛选来获取 - - # # 小航富途 - # market_data_hang = read_missing_codes_basic() - # - - # market_data = list(set(market_data_all) - set(market_data_hang)) - - nCount = 0 # 记录账号获取多少只股票的数据 - market_data_all = get_stock_codes() - for code in market_data_all: - stock_hk_index_daily_sina_df = ak.stock_hk_index_daily_sina(symbol=code[3:]) - custom_table_name = 'hk_' + code[3:] # 自定义表名 - success = save_quotes_to_db(db_config, stock_hk_index_daily_sina_df, table_name=custom_table_name) - time.sleep(5) - - - diff --git a/config/HK_futu.txt b/config/HK_futu.txt new file mode 100644 index 0000000..79dafa6 --- /dev/null +++ b/config/HK_futu.txt @@ -0,0 +1,892 @@ +HK.01193 +HK.00610 +HK.00548 +HK.00496 +HK.02405 +HK.00899 +HK.02215 +HK.00353 +HK.00805 +HK.02352 +HK.00266 +HK.01068 +HK.01884 +HK.01771 +HK.01953 +HK.02908 +HK.02519 +HK.00604 +HK.01756 +HK.02330 +HK.01916 +HK.00902 +HK.02329 +HK.00954 +HK.00860 +HK.01215 +HK.01072 +HK.01631 +HK.00564 +HK.00829 +HK.01750 +HK.01523 +HK.01448 +HK.01080 +HK.02218 +HK.00565 +HK.01758 +HK.00418 +HK.00798 +HK.00413 +HK.00830 +HK.01635 +HK.02219 +HK.02528 +HK.00018 +HK.00367 +HK.00848 +HK.00160 +HK.01379 +HK.00021 +HK.02076 +HK.01013 +HK.01486 +HK.01596 +HK.02325 +HK.00147 +HK.01024 +HK.00335 +HK.00941 +HK.01221 +HK.02221 +HK.01508 +HK.00743 +HK.00804 +HK.02033 +HK.00921 +HK.02425 +HK.01741 +HK.00341 +HK.02627 +HK.01228 +HK.02613 +HK.01967 +HK.01489 +HK.02336 +HK.00630 +HK.00733 +HK.02016 +HK.01729 +HK.01346 +HK.00365 +HK.00857 +HK.02331 +HK.00756 +HK.01712 +HK.00305 +HK.00227 +HK.01412 +HK.01970 +HK.02322 +HK.00189 +HK.01661 +HK.01803 +HK.01842 +HK.00667 +HK.00211 +HK.01113 +HK.01652 +HK.02411 +HK.01205 +HK.01860 +HK.00619 +HK.00553 +HK.00607 +HK.00770 +HK.02279 +HK.02489 +HK.01211 +HK.01878 +HK.02453 +HK.00458 +HK.00450 +HK.00580 +HK.03319 +HK.01575 +HK.00839 +HK.00242 +HK.01278 +HK.01718 +HK.01239 +HK.01952 +HK.00287 +HK.02321 +HK.01812 +HK.00841 +HK.00025 +HK.00508 +HK.01555 +HK.01683 +HK.02409 +HK.00004 +HK.00185 +HK.00695 +HK.02196 +HK.00086 +HK.00788 +HK.01907 +HK.02007 +HK.00661 +HK.00103 +HK.00412 +HK.01826 +HK.00280 +HK.01429 +HK.01738 +HK.00115 +HK.00173 +HK.01882 +HK.01961 +HK.01152 +HK.01851 +HK.00840 +HK.01427 +HK.00924 +HK.00918 +HK.00685 +HK.01334 +HK.01245 +HK.01480 +HK.00711 +HK.00061 +HK.02728 +HK.01262 +HK.01898 +HK.00148 +HK.00323 +HK.00658 +HK.03326 +HK.02904 +HK.00834 +HK.01666 +HK.01235 +HK.01788 +HK.00934 +HK.00581 +HK.01319 +HK.01983 +HK.01335 +HK.02588 +HK.01830 +HK.02913 +HK.02440 +HK.00377 +HK.01176 +HK.01303 +HK.00524 +HK.00498 +HK.02193 +HK.02727 +HK.00859 +HK.02107 +HK.00688 +HK.00730 +HK.02269 +HK.01628 +HK.00451 +HK.02423 +HK.02315 +HK.01477 +HK.01413 +HK.00369 +HK.00927 +HK.00432 +HK.00220 +HK.00040 +HK.00499 +HK.00046 +HK.01119 +HK.01759 +HK.02608 +HK.01982 +HK.02167 +HK.03309 +HK.01908 +HK.00222 +HK.00529 +HK.02057 +HK.01643 +HK.01234 +HK.01380 +HK.01202 +HK.00200 +HK.00999 +HK.01098 +HK.01125 +HK.02338 +HK.00914 +HK.01395 +HK.01772 +HK.01686 +HK.01958 +HK.01905 +HK.01673 +HK.00019 +HK.01945 +HK.01841 +HK.00272 +HK.00055 +HK.02488 +HK.02078 +HK.00331 +HK.00907 +HK.00252 +HK.00719 +HK.01165 +HK.01861 +HK.01986 +HK.00547 +HK.00180 +HK.02317 +HK.00213 +HK.00728 +HK.01244 +HK.02540 +HK.01359 +HK.00144 +HK.01720 +HK.02297 +HK.01736 +HK.02510 +HK.00974 +HK.01326 +HK.00069 +HK.01875 +HK.01138 +HK.01284 +HK.00399 +HK.01675 +HK.02025 +HK.00864 +HK.02048 +HK.00010 +HK.00526 +HK.01599 +HK.01055 +HK.00358 +HK.01302 +HK.01378 +HK.02415 +HK.00623 +HK.02521 +HK.00579 +HK.01766 +HK.03336 +HK.03368 +HK.01442 +HK.02427 +HK.01200 +HK.01183 +HK.00897 +HK.02399 +HK.03369 +HK.01046 +HK.02333 +HK.01757 +HK.00178 +HK.00177 +HK.00968 +HK.00146 +HK.00271 +HK.02361 +HK.02507 +HK.00611 +HK.01118 +HK.00111 +HK.00997 +HK.01145 +HK.01580 +HK.00291 +HK.00811 +HK.01501 +HK.01115 +HK.01258 +HK.02157 +HK.01831 +HK.01939 +HK.00947 +HK.01481 +HK.02688 +HK.01672 +HK.02028 +HK.00754 +HK.01833 +HK.00374 +HK.02097 +HK.00727 +HK.00484 +HK.01796 +HK.02878 +HK.02356 +HK.02326 +HK.02172 +HK.00357 +HK.01500 +HK.01036 +HK.01618 +HK.01217 +HK.01568 +HK.01742 +HK.00936 +HK.02469 +HK.00003 +HK.01084 +HK.01719 +HK.01112 +HK.01717 +HK.00853 +HK.00355 +HK.00586 +HK.01545 +HK.03332 +HK.00236 +HK.00164 +HK.00679 +HK.00078 +HK.02438 +HK.00866 +HK.01701 +HK.01529 +HK.01238 +HK.01009 +HK.01127 +HK.02299 +HK.01408 +HK.03323 +HK.00691 +HK.01991 +HK.02385 +HK.02623 +HK.02506 +HK.01615 +HK.02533 +HK.00118 +HK.01241 +HK.00267 +HK.00320 +HK.01045 +HK.00065 +HK.00298 +HK.00606 +HK.02223 +HK.00348 +HK.01201 +HK.00133 +HK.02002 +HK.02490 +HK.02512 +HK.00422 +HK.02502 +HK.00379 +HK.01133 +HK.01476 +HK.00012 +HK.02392 +HK.00201 +HK.00258 +HK.02226 +HK.00666 +HK.02250 +HK.00089 +HK.01897 +HK.01050 +HK.00013 +HK.00138 +HK.00709 +HK.00325 +HK.02481 +HK.01822 +HK.01292 +HK.01739 +HK.02121 +HK.00145 +HK.00471 +HK.01104 +HK.01682 +HK.02372 +HK.00693 +HK.00036 +HK.01247 +HK.01637 +HK.00595 +HK.02050 +HK.00120 +HK.01780 +HK.00423 +HK.01005 +HK.02531 +HK.00079 +HK.00875 +HK.01846 +HK.02235 +HK.01869 +HK.00772 +HK.00588 +HK.01832 +HK.01162 +HK.01349 +HK.02360 +HK.01209 +HK.01698 +HK.00826 +HK.03321 +HK.01460 +HK.03329 +HK.00097 +HK.02669 +HK.00032 +HK.00361 +HK.01269 +HK.01579 +HK.00098 +HK.02605 +HK.01870 +HK.01090 +HK.01419 +HK.00396 +HK.00101 +HK.00557 +HK.01170 +HK.01483 +HK.00419 +HK.00297 +HK.00675 +HK.02298 +HK.02246 +HK.03348 +HK.02682 +HK.01760 +HK.01696 +HK.03316 +HK.00159 +HK.00410 +HK.00764 +HK.00276 +HK.01993 +HK.02899 +HK.00983 +HK.01571 +HK.01521 +HK.00876 +HK.02252 +HK.00909 +HK.01252 +HK.01655 +HK.01856 +HK.01063 +HK.00966 +HK.02386 +HK.02113 +HK.00235 +HK.00096 +HK.01930 +HK.01382 +HK.02465 +HK.02616 +HK.00533 +HK.00194 +HK.00387 +HK.01163 +HK.01317 +HK.01941 +HK.01259 +HK.00090 +HK.00268 +HK.00519 +HK.00517 +HK.01726 +HK.01943 +HK.00874 +HK.01669 +HK.01660 +HK.00128 +HK.00216 +HK.01179 +HK.00844 +HK.01283 +HK.00038 +HK.01802 +HK.01330 +HK.01817 +HK.00932 +HK.02592 +HK.00229 +HK.00938 +HK.01975 +HK.00510 +HK.02696 +HK.01289 +HK.00975 +HK.00833 +HK.02251 +HK.00686 +HK.03377 +HK.01667 +HK.00854 +HK.00151 +HK.00261 +HK.00827 +HK.01420 +HK.02101 +HK.00388 +HK.01516 +HK.00400 +HK.01522 +HK.00898 +HK.01662 +HK.01663 +HK.01288 +HK.02339 +HK.01186 +HK.00296 +HK.00896 +HK.00382 +HK.02576 +HK.00760 +HK.00154 +HK.02125 +HK.01836 +HK.02383 +HK.00929 +HK.01576 +HK.00560 +HK.01636 +HK.01405 +HK.01606 +HK.02391 +HK.01061 +HK.00670 +HK.02589 +HK.01062 +HK.00347 +HK.00168 +HK.02482 +HK.01999 +HK.02919 +HK.01347 +HK.00480 +HK.01723 +HK.01373 +HK.01612 +HK.02439 +HK.02225 +HK.01910 +HK.00426 +HK.01695 +HK.01730 +HK.01608 +HK.02228 +HK.01948 +HK.01583 +HK.00370 +HK.02310 +HK.02609 +HK.01816 +HK.00230 +HK.01181 +HK.00277 +HK.01978 +HK.01058 +HK.00306 +HK.01617 +HK.02306 +HK.01777 +HK.02499 +HK.01947 +HK.00132 +HK.01787 +HK.01586 +HK.01091 +HK.01795 +HK.02552 +HK.02881 +HK.01735 +HK.01671 +HK.01339 +HK.02160 +HK.02003 +HK.02108 +HK.00029 +HK.02450 +HK.01421 +HK.01148 +HK.01632 +HK.01282 +HK.01653 +HK.00913 +HK.00568 +HK.01361 +HK.02255 +HK.00301 +HK.03320 +HK.00794 +HK.00694 +HK.00911 +HK.01578 +HK.01903 +HK.00042 +HK.01800 +HK.02455 +HK.00333 +HK.00994 +HK.00217 +HK.00755 +HK.01731 +HK.00060 +HK.01962 +HK.00030 +HK.02161 +HK.00536 +HK.00995 +HK.01008 +HK.01681 +HK.02198 +HK.01446 +HK.00199 +HK.00629 +HK.00228 +HK.02501 +HK.01450 +HK.01951 +HK.02237 +HK.03302 +HK.00150 +HK.02008 +HK.02518 +HK.01280 +HK.02222 +HK.02105 +HK.00488 +HK.00119 +HK.02138 +HK.01132 +HK.00226 +HK.02498 +HK.00554 +HK.02648 +HK.00248 +HK.00945 +HK.00746 +HK.00978 +HK.00990 +HK.01883 +HK.01105 +HK.00558 +HK.01937 +HK.00411 +HK.00131 +HK.02486 +HK.01206 +HK.00076 +HK.00384 +HK.00218 +HK.00075 +HK.00894 +HK.01691 +HK.02917 +HK.00052 +HK.02789 +HK.01452 +HK.01808 +HK.01940 +HK.01773 +HK.02013 +HK.02601 +HK.02877 +HK.01114 +HK.00493 +HK.00033 +HK.02212 +HK.01108 +HK.00926 +HK.01499 +HK.01195 +HK.01622 +HK.00139 +HK.01561 +HK.01525 +HK.01354 +HK.00311 +HK.00725 +HK.00057 +HK.02565 +HK.00620 +HK.00001 +HK.01065 +HK.01153 +HK.03322 +HK.02571 +HK.00483 +HK.00169 +HK.00632 +HK.01917 +HK.00817 +HK.01271 +HK.02402 +HK.01857 +HK.00527 +HK.01047 +HK.02271 +HK.00993 +HK.02369 +HK.02086 +HK.00129 +HK.00683 +HK.02137 +HK.00084 +HK.01315 +HK.01845 +HK.00540 +HK.00135 +HK.01727 +HK.01909 +HK.00659 +HK.00636 +HK.01927 +HK.00026 +HK.00163 +HK.00832 +HK.01044 +HK.01225 +HK.02152 +HK.02509 +HK.02536 +HK.01472 +HK.00285 +HK.01029 +HK.01129 +HK.01538 +HK.02208 +HK.01355 +HK.02892 +HK.02233 +HK.00391 +HK.01370 +HK.02363 +HK.00439 +HK.00476 +HK.00070 +HK.01640 +HK.00807 +HK.00500 +HK.00110 +HK.01888 +HK.02158 +HK.01936 +HK.01858 +HK.02555 +HK.01323 +HK.00386 +HK.01038 +HK.00559 +HK.00888 +HK.00883 +HK.00352 +HK.02559 +HK.00836 +HK.00408 +HK.01496 +HK.00294 +HK.00058 +HK.00682 +HK.01224 +HK.02182 +HK.00697 +HK.00819 +HK.00626 +HK.01173 +HK.01515 +HK.01146 +HK.00721 +HK.02600 +HK.00660 +HK.02031 +HK.02012 +HK.02178 +HK.00123 +HK.00340 +HK.01713 +HK.00784 +HK.01375 +HK.03313 +HK.00992 +HK.01502 +HK.00551 +HK.00837 +HK.00919 +HK.01075 +HK.02503 +HK.00912 +HK.01079 +HK.00895 +HK.01447 +HK.01203 +HK.02689 +HK.01906 +HK.00345 +HK.01362 +HK.02865 +HK.02270 +HK.00406 +HK.01920 +HK.01708 +HK.02407 +HK.01848 +HK.02176 +HK.02135 +HK.00863 +HK.01030 +HK.01039 +HK.02436 +HK.00814 +HK.01313 +HK.00474 +HK.02096 +HK.01286 +HK.00351 +HK.01070 +HK.01931 +HK.01198 +HK.00191 +HK.00245 +HK.02459 +HK.00777 +HK.01965 +HK.01268 +HK.00051 +HK.01548 +HK.01321 +HK.01623 +HK.01572 +HK.02515 +HK.01177 +HK.01949 +HK.02185 +HK.01281 +HK.00219 +HK.02260 +HK.01868 +HK.02529 +HK.02522 +HK.01001 +HK.02307 +HK.03318 +HK.00900 +HK.02238 +HK.00127 +HK.00053 +HK.01417 +HK.00300 +HK.02563 +HK.01591 +HK.01715 \ No newline at end of file diff --git a/config/hang_futu.txt b/config/hang_futu.txt index a7d199d..8839f0d 100644 --- a/config/hang_futu.txt +++ b/config/hang_futu.txt @@ -698,4 +698,6 @@ HK.02923 HK.02925 HK.02926 HK.02927 -HK.02929 \ No newline at end of file +HK.02929 +HK.02631 +HK.02930 \ No newline at end of file diff --git a/config/备注.txt b/config/备注.txt index 053e879..85e831e 100644 --- a/config/备注.txt +++ b/config/备注.txt @@ -1,6 +1,6 @@ kevin_futu: - 已使用1000行情 + 已使用 1000 行情 hang_futu: - 已使用701行情 + 已使用 703 行情 HK牛仔： - 已使用893行情 \ No newline at end of file + 已使用 893 行情 \ No newline at end of file