DdStone/HKDataManagment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update exportdata

Browse Source
This commit is contained in:
unknown
2025-09-03 21:41:11 +08:00
parent 41beb2ec33
commit 7d35766a09
13 changed files with 734 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
DataAnalysis/DataExporter.py
View File

@@ -143,10 +143,54 @@ class DataExporter:
self.logger.error("无法获取月度均价数据")
return False
# 读取港股通标记
hk_inout_data = self.get_hk_inout()
for item in monthly_data:
stock_name = item.get('stock_code')[3:]
if stock_name in hk_inout_data:
item['in_out'] = 1
else:
item['in_out'] = 0
# 导出结果
file_path = 'data/' + csv_file if csv_file else None
csv_success = True
if csv_file:
csv_success = self.export_to_csv(monthly_data, file_path)
return csv_success
return csv_success
def get_hk_inout(self) -> Optional[List[Dict]]:
"""
从conditionalselection表读取流通股本数据
Args:
table_name: 源数据表名
Returns:
List[Dict]: 查询结果数据集，失败返回None
"""
try:
with MySQLHelper(**self.db_config) as db:
# 查询流通股本数据
data = db.execute_query(f"""
SELECT stock_code
FROM hk_stock_connect
WHERE in_out = '1'
ORDER BY stock_code
""")
if not data:
self.logger.error(f"获取数据失败")
return None
return [
row['stock_code']
for row in data
if row['stock_code']
]
except Exception as e:
self.logger.error(f"从数据库读取流通股本数据失败: {str(e)}")
return None

160
DataAnalysis/MarketDataCalculator.py
View File

@@ -16,7 +16,8 @@ from typing import Optional, List, Dict, Union, Tuple
import csv
from typing import List, Dict, Optional
from datetime import datetime
from base import LogHelper, MySQLHelper, Config
from base import LogHelper, MySQLHelper, ConfigInfo
class MarketDataCalculator:
"""
@@ -41,8 +42,8 @@ class MarketDataCalculator:
"""
self.db_config = db_config
self.logger = LogHelper(logger_name=logger_name).setup()
self.month_ranges = Config.ConfigInfo.MONTH_RANGES
self.head_map = Config.ConfigInfo.HEADER_MAP
self.month_ranges = ConfigInfo.MONTH_RANGES
self.head_map = ConfigInfo.HEADER_MAP
def create_monthly_avg_table(self, target_table: str = "monthly_close_avg") -> bool:
"""
@@ -69,6 +70,7 @@ class MarketDataCalculator:
ym_2506 DECIMAL(20, 5) COMMENT '2025年06月',
ym_2507 DECIMAL(20, 5) COMMENT '2025年07月',
ym_2508 DECIMAL(20, 5) COMMENT '2025年08月',
ym_2509 DECIMAL(20, 5) COMMENT '2025年09月',
avg_all DECIMAL(20, 5) COMMENT '月间均值',
update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
UNIQUE KEY uk_stock_code (stock_code)
@@ -82,7 +84,7 @@ class MarketDataCalculator:
return False
def calculate_and_save_monthly_avg(self,
source_table: str = "stock_quotes",
stock_code: str = "code",
target_table: str = "monthly_close_avg") -> bool:
"""
计算并保存2024年10月至2025年8月的月均流通市值
@@ -100,79 +102,85 @@ class MarketDataCalculator:
return False
with MySQLHelper(**self.db_config) as db:
# 获取所有股票代码和名称
stock_info = db.execute_query(
f"SELECT DISTINCT stock_code, stock_name FROM {source_table}"
)
if not stock_info:
self.logger.error("没有获取到股票基本信息")
return False
# 为每只股票计算各月均值
for stock in stock_info:
stock_code = stock['stock_code']
stock_name = stock['stock_name']
monthly_data = {'stock_code': stock_code, 'stock_name': stock_name}
# 计算每个月的均值
for month_col, (start_date, end_date) in self.month_ranges.items():
sql = """
SELECT AVG(close_price * float_share) as avg_close
FROM {}
WHERE stock_code = %s
AND trade_date BETWEEN %s AND %s
AND close_price IS NOT NULL
AND float_share IS NOT NULL
""".format(source_table)
result = db.execute_query(sql, (stock_code, start_date, end_date))
# 保存小数点后两位，以亿为单位
# monthly_data[month_col] = float(result[0]['avg_close']) * 1000 if result and result[0]['avg_close'] else None
monthly_data[month_col] = round(float(result[0]['avg_close']) * 1000 / 100000000, 3) if result and result[0]['avg_close'] else None
# 提取所有以 'ym_' 开头的键的值
ym_values = [value for key, value in monthly_data.items() if key.startswith('ym_')]
valid_ym_values = [value for value in ym_values if value is not None]
# 计算全部月的均值
if valid_ym_values:
average = sum(valid_ym_values) / len(valid_ym_values)
monthly_data['avg_all'] = average
self.logger.debug(f"股票 {stock_code} 月间流通市值平均值为: {average}")
else:
monthly_data['avg_all'] = 0 # 给一个空值，保证数据库不报错
self.logger.warning(f"股票 {stock_code} 没有有效的月度数据")
# 插入或更新数据
upsert_sql = f"""
INSERT INTO {target_table} (
stock_code, stock_name,
ym_2501, ym_2502, ym_2503, ym_2504,
ym_2505, ym_2506,ym_2507, ym_2508,
avg_all
) VALUES (
%(stock_code)s, %(stock_name)s,
%(ym_2501)s, %(ym_2502)s, %(ym_2503)s, %(ym_2504)s,
%(ym_2505)s, %(ym_2506)s, %(ym_2507)s, %(ym_2508)s,
%(avg_all)s
)
ON DUPLICATE KEY UPDATE
stock_name = VALUES(stock_name),
ym_2501 = VALUES(ym_2501),
ym_2502 = VALUES(ym_2502),
ym_2503 = VALUES(ym_2503),
ym_2504 = VALUES(ym_2504),
ym_2505 = VALUES(ym_2505),
ym_2506 = VALUES(ym_2506),
ym_2507 = VALUES(ym_2507),
ym_2508 = VALUES(ym_2508),
avg_all = VALUES(avg_all),
update_time = CURRENT_TIMESTAMP
sql = """
SELECT stock_code, stock_name
FROM stock_filter
WHERE stock_code = %s
"""
db.execute_update(upsert_sql, monthly_data)
# stock_filter表格可以当成标准表
stock_info = db.execute_query(sql,(stock_code))
if len(stock_info) == 0:
return
stock_code = stock_info[0]['stock_code']
stock_name = stock_info[0]['stock_name']
monthly_data = {'stock_code': stock_code, 'stock_name': stock_name}
# 计算每个月的均值
source_table = 'hk_' + stock_code[3:]
for month_col, (start_date, end_date) in self.month_ranges.items():
sql = """
SELECT AVG(close_price * float_share) as avg_close
FROM {}
WHERE stock_code = %s
AND trade_date BETWEEN %s AND %s
AND close_price IS NOT NULL
AND float_share IS NOT NULL
""".format(source_table)
result = db.execute_query(sql, (stock_code, start_date, end_date))
# 保存小数点后两位，以亿为单位
# monthly_data[month_col] = float(result[0]['avg_close']) * 1000 if result and result[0]['avg_close'] else None
monthly_data[month_col] = round(float(result[0]['avg_close']) * 1000 / 100000000, 3) if result and result[0]['avg_close'] else None
# 提取所有以 'ym_' 开头的键的值
ym_values = [value for key, value in monthly_data.items() if key.startswith('ym_')]
valid_ym_values = [value for value in ym_values if value is not None]
# 计算全部月的均值
if valid_ym_values:
average = sum(valid_ym_values) / len(valid_ym_values)
monthly_data['avg_all'] = average
self.logger.debug(f"股票 {stock_code} 月间流通市值平均值为: {average}")
else:
monthly_data['avg_all'] = 0 # 给一个空值，保证数据库不报错
self.logger.warning(f"股票 {stock_code} 没有有效的月度数据")
# 插入或更新数据
upsert_sql = f"""
INSERT INTO {target_table} (
stock_code, stock_name,
ym_2501, ym_2502, ym_2503, ym_2504,
ym_2505, ym_2506,ym_2507, ym_2508,
ym_2509,
avg_all
) VALUES (
%(stock_code)s, %(stock_name)s,
%(ym_2501)s, %(ym_2502)s, %(ym_2503)s, %(ym_2504)s,
%(ym_2505)s, %(ym_2506)s, %(ym_2507)s, %(ym_2508)s,
%(ym_2509)s,
%(avg_all)s
)
ON DUPLICATE KEY UPDATE
stock_name = VALUES(stock_name),
ym_2501 = VALUES(ym_2501),
ym_2502 = VALUES(ym_2502),
ym_2503 = VALUES(ym_2503),
ym_2504 = VALUES(ym_2504),
ym_2505 = VALUES(ym_2505),
ym_2506 = VALUES(ym_2506),
ym_2507 = VALUES(ym_2507),
ym_2508 = VALUES(ym_2508),
ym_2509 = VALUES(ym_2509),
avg_all = VALUES(avg_all),
update_time = CURRENT_TIMESTAMP
"""
db.execute_update(upsert_sql, monthly_data)
# self.logger.info("月度均值计算和保存完成")
return True
except Exception as e:
@@ -435,4 +443,6 @@ class MarketDataCalculator:
if csv_file:
csv_success = self.export_to_csv(monthly_data, file_path)
return csv_success
return csv_success