update flies
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -264,6 +264,4 @@ compile_commands.json
|
|||||||
|
|
||||||
*_qmlcache.qrc
|
*_qmlcache.qrc
|
||||||
|
|
||||||
|
logs/
|
||||||
data/
|
|
||||||
log/
|
|
||||||
|
|||||||
89
LogHelper.py
89
LogHelper.py
@@ -1,63 +1,90 @@
|
|||||||
|
"""
|
||||||
|
日志管理类
|
||||||
|
|
||||||
|
日志输出到运行目录指定文件夹
|
||||||
|
日志文件按照:YYYY-MM-dd.log 格式输出
|
||||||
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import sys
|
import sys
|
||||||
|
import os
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
class LogHelper:
|
class LogHelper:
|
||||||
|
# 内部日志级别映射表
|
||||||
|
_LEVEL_MAP = {
|
||||||
|
'DEBUG': 10,
|
||||||
|
'INFO': 20,
|
||||||
|
'WARNING': 30,
|
||||||
|
'ERROR': 40,
|
||||||
|
'CRITICAL': 50
|
||||||
|
}
|
||||||
|
|
||||||
def __init__(self,
|
def __init__(self,
|
||||||
level=logging.INFO,
|
level='INFO',
|
||||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||||
handlers=None):
|
handlers=None,
|
||||||
|
log_dir="logs",
|
||||||
|
logger_name=None):
|
||||||
"""
|
"""
|
||||||
初始化日志配置
|
初始化日志配置
|
||||||
|
|
||||||
:param level: 日志级别,默认为 logging.INFO
|
:param level: 日志级别,默认为 logging.INFO
|
||||||
:param format: 日志格式字符串
|
:param format: 日志格式字符串
|
||||||
:param handlers: 日志处理器列表,默认为空(会自动添加控制台处理器)
|
:param handlers: 日志处理器列表,默认为空(会自动添加控制台处理器)
|
||||||
|
:param log_dir: 日志存储目录,默认为"logs"
|
||||||
"""
|
"""
|
||||||
self.level = level
|
self.level = self._LEVEL_MAP.get(level.upper(), 20) # 默认INFO
|
||||||
self.format = format
|
self.format = format
|
||||||
self.handlers = handlers if handlers is not None else []
|
self.handlers = handlers if handlers is not None else []
|
||||||
|
self.log_dir = log_dir
|
||||||
|
self.logger_name = logger_name
|
||||||
|
# 确保日志目录存在
|
||||||
|
os.makedirs(self.log_dir, exist_ok=True)
|
||||||
|
|
||||||
def add_console_handler(self, stream=sys.stdout, encoding='utf-8'):
|
def add_console_handler(self, stream=sys.stdout, encoding='utf-8'):
|
||||||
"""添加控制台处理器"""
|
"""添加控制台处理器"""
|
||||||
console_handler = logging.StreamHandler(stream)
|
console_handler = logging.StreamHandler(stream)
|
||||||
console_handler.setFormatter(logging.Formatter(self.format))
|
console_handler.setFormatter(logging.Formatter(self.format))
|
||||||
# 设置控制台编码
|
|
||||||
if encoding:
|
if encoding:
|
||||||
console_handler.encoding = encoding
|
console_handler.encoding = encoding
|
||||||
self.handlers.append(console_handler)
|
self.handlers.append(console_handler)
|
||||||
|
return self # 支持链式调用
|
||||||
|
|
||||||
def add_file_handler(self, filename, encoding='utf-8'):
|
def _get_daily_log_path(self):
|
||||||
"""添加文件处理器(解决中文乱码问题)"""
|
"""生成基于当前日期的日志文件路径"""
|
||||||
# 使用支持UTF-8编码的文件处理器
|
today = datetime.now().strftime("%Y-%m-%d")
|
||||||
filepath = "log/" + filename
|
filename = f"{today}.log"
|
||||||
file_handler = logging.FileHandler(filepath, encoding=encoding)
|
return os.path.join(self.log_dir, filename)
|
||||||
|
|
||||||
|
def _add_daily_file_handler(self, encoding='utf-8'):
|
||||||
|
"""添加按日期自动创建的文件处理器"""
|
||||||
|
log_path = self._get_daily_log_path()
|
||||||
|
file_handler = logging.FileHandler(log_path, encoding=encoding)
|
||||||
file_handler.setFormatter(logging.Formatter(self.format))
|
file_handler.setFormatter(logging.Formatter(self.format))
|
||||||
self.handlers.append(file_handler)
|
self.handlers.append(file_handler)
|
||||||
|
return self # 支持链式调用
|
||||||
|
|
||||||
def setup(self):
|
def setup(self):
|
||||||
"""应用日志配置"""
|
"""应用日志配置(自动添加日期文件处理器)"""
|
||||||
logging.basicConfig(
|
# 如果未提供任何处理器,默认添加控制台和日期文件处理器
|
||||||
level=self.level,
|
if not self.handlers:
|
||||||
format=self.format,
|
self.add_console_handler()
|
||||||
handlers=self.handlers
|
self._add_daily_file_handler()
|
||||||
)
|
|
||||||
|
|
||||||
# # 使用示例
|
# 获取或创建logger
|
||||||
# if __name__ == "__main__":
|
logger = logging.getLogger(self.logger_name)
|
||||||
# # 创建配置实例
|
logger.setLevel(self.level)
|
||||||
# logger_config = LoggerConfig(
|
|
||||||
# level=logging.DEBUG, # 设置日志级别为 DEBUG
|
|
||||||
# format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
|
||||||
# )
|
|
||||||
|
|
||||||
# # # 添加处理器
|
# 移除所有现有处理器(避免重复添加)
|
||||||
# logger_config.add_console_handler() # 默认输出到 stdout
|
for handler in logger.handlers[:]:
|
||||||
# logger_config.add_file_handler('Debug.log') # 添加文件日志
|
logger.removeHandler(handler)
|
||||||
|
|
||||||
# # # 应用配置
|
# 添加配置的处理器
|
||||||
# logger_config.setup()
|
for handler in self.handlers:
|
||||||
|
logger.addHandler(handler)
|
||||||
|
|
||||||
# # 测试日志
|
# 确保日志消息不会传递给父logger(避免重复记录)
|
||||||
# logging.debug("Debug 信息")
|
logger.propagate = False
|
||||||
# logging.info("Info 信息")
|
|
||||||
# logging.warning("警告信息")
|
return logger
|
||||||
@@ -3,27 +3,24 @@
|
|||||||
—— 增加事务管理
|
—— 增加事务管理
|
||||||
—— 增加ID获取
|
—— 增加ID获取
|
||||||
—— 增加表操作等使用功能
|
—— 增加表操作等使用功能
|
||||||
|
—— 数据使用过程中出现异常时,才输出日志
|
||||||
"""
|
"""
|
||||||
import pymysql
|
import pymysql
|
||||||
from pymysql import Error
|
from pymysql import Error
|
||||||
from typing import List, Dict, Union, Optional, Tuple
|
from typing import List, Dict, Union, Optional, Tuple
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from LogHelper import LogHelper
|
from LogHelper import LogHelper
|
||||||
import logging
|
|
||||||
|
|
||||||
# 创建配置实例
|
# 基本用法(自动创建日期日志+控制台输出)
|
||||||
logHelper = LogHelper(
|
logger = LogHelper(logger_name = 'database').setup()
|
||||||
level=logging.DEBUG, # 设置日志级别为 DEBUG
|
|
||||||
format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
|
||||||
)
|
|
||||||
|
|
||||||
# # 添加处理器
|
# # 高级用法(自定义配置)
|
||||||
logHelper.add_console_handler() # 默认输出到 stdout
|
# logger = LogHelper(
|
||||||
logHelper.add_file_handler('Debug.log') # 添加文件日志
|
# level=logging.DEBUG,
|
||||||
|
# log_dir="databaselogs",
|
||||||
|
# format='%(levelname)s - %(message)s'
|
||||||
|
# ).setup()
|
||||||
|
|
||||||
# # 应用配置
|
|
||||||
logHelper.setup()
|
|
||||||
logger = logging.getLogger('StockDataImporter')
|
|
||||||
|
|
||||||
class MySQLHelper:
|
class MySQLHelper:
|
||||||
def __init__(self, host: str, user: str, password: str, database: str,
|
def __init__(self, host: str, user: str, password: str, database: str,
|
||||||
@@ -62,7 +59,7 @@ class MySQLHelper:
|
|||||||
cursorclass=pymysql.cursors.DictCursor # 返回字典形式的结果
|
cursorclass=pymysql.cursors.DictCursor # 返回字典形式的结果
|
||||||
)
|
)
|
||||||
self.cursor = self.connection.cursor()
|
self.cursor = self.connection.cursor()
|
||||||
logger.info("MySQL数据库连接成功")
|
# logger.info("MySQL数据库连接成功")
|
||||||
return True
|
return True
|
||||||
except Error as e:
|
except Error as e:
|
||||||
logger.error(f"连接MySQL数据库失败: {e}")
|
logger.error(f"连接MySQL数据库失败: {e}")
|
||||||
@@ -76,7 +73,7 @@ class MySQLHelper:
|
|||||||
self.cursor.close()
|
self.cursor.close()
|
||||||
if self.connection:
|
if self.connection:
|
||||||
self.connection.close()
|
self.connection.close()
|
||||||
logger.info("MySQL数据库连接已关闭")
|
# logger.info("MySQL数据库连接已关闭")
|
||||||
|
|
||||||
def execute_query(self, sql: str, params: Union[Tuple, List, Dict, None] = None) -> List[Dict]:
|
def execute_query(self, sql: str, params: Union[Tuple, List, Dict, None] = None) -> List[Dict]:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
|
"""
|
||||||
|
使用 baostock 数数据源获取/更新A股数据
|
||||||
|
|
||||||
|
***
|
||||||
|
该数据源中会返回非交易日数据,导致数据存储异常
|
||||||
|
"""
|
||||||
from MySQLHelper import MySQLHelper
|
from MySQLHelper import MySQLHelper
|
||||||
from LogHelper import LogHelper
|
from LogHelper import LogHelper
|
||||||
import logging
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
@@ -8,16 +13,8 @@ from datetime import datetime, timedelta
|
|||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
import baostock as bs
|
import baostock as bs
|
||||||
|
|
||||||
# 创建配置实例
|
# 基本用法(自动创建日期日志+控制台输出)
|
||||||
logHelper = LogHelper(
|
logger = LogHelper(logger_name = 'baoStock').setup()
|
||||||
level=logging.DEBUG,
|
|
||||||
format='%(asctime)s [%(levelname)s] %(message)s'
|
|
||||||
)
|
|
||||||
|
|
||||||
logHelper.add_console_handler()
|
|
||||||
logHelper.add_file_handler('Debug.log')
|
|
||||||
logHelper.setup()
|
|
||||||
logger = logging.getLogger('StockDataImporter')
|
|
||||||
|
|
||||||
# 数据库配置
|
# 数据库配置
|
||||||
DB_CONFIG = {
|
DB_CONFIG = {
|
||||||
@@ -100,7 +97,7 @@ def create_stock_table(db: MySQLHelper, table_name: str) -> bool:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
db.execute_update(create_table_sql)
|
db.execute_update(create_table_sql)
|
||||||
logger.info(f"成功创建表: {table_name}")
|
logger.info(f"成功打开表: {table_name}")
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"创建表 {table_name} 失败: {e}")
|
logger.error(f"创建表 {table_name} 失败: {e}")
|
||||||
|
|||||||
2871
data/A股列表.csv
Normal file
2871
data/A股列表.csv
Normal file
File diff suppressed because it is too large
Load Diff
BIN
data/A股列表.xlsx
Normal file
BIN
data/A股列表.xlsx
Normal file
Binary file not shown.
1695
data/GPLIST.csv
Normal file
1695
data/GPLIST.csv
Normal file
File diff suppressed because it is too large
Load Diff
BIN
data/GPLIST.xlsx
Normal file
BIN
data/GPLIST.xlsx
Normal file
Binary file not shown.
@@ -8,7 +8,6 @@
|
|||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import os
|
import os
|
||||||
import logging
|
|
||||||
import sys
|
import sys
|
||||||
import csv
|
import csv
|
||||||
import chardet # 用于检测文件编码
|
import chardet # 用于检测文件编码
|
||||||
@@ -18,19 +17,7 @@ from datetime import datetime
|
|||||||
from MySQLHelper import MySQLHelper
|
from MySQLHelper import MySQLHelper
|
||||||
from LogHelper import LogHelper
|
from LogHelper import LogHelper
|
||||||
|
|
||||||
# 创建配置实例
|
logger = LogHelper(logger_name = 'SH_Import').setup()
|
||||||
logHelper = LogHelper(
|
|
||||||
level=logging.DEBUG, # 设置日志级别为 DEBUG
|
|
||||||
format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
|
||||||
)
|
|
||||||
|
|
||||||
# # 添加处理器
|
|
||||||
logHelper.add_console_handler() # 默认输出到 stdout
|
|
||||||
logHelper.add_file_handler('Debug.log') # 添加文件日志
|
|
||||||
|
|
||||||
# # 应用配置
|
|
||||||
logHelper.setup()
|
|
||||||
logger = logging.getLogger('StockDataImporter')
|
|
||||||
|
|
||||||
class StockDataImporter:
|
class StockDataImporter:
|
||||||
"""股票数据导入工具(支持CSV)"""
|
"""股票数据导入工具(支持CSV)"""
|
||||||
|
|||||||
@@ -11,25 +11,12 @@ from MySQLHelper import MySQLHelper
|
|||||||
from LogHelper import LogHelper
|
from LogHelper import LogHelper
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import os
|
import os
|
||||||
import logging
|
|
||||||
import sys
|
import sys
|
||||||
import csv
|
import csv
|
||||||
import chardet
|
import chardet
|
||||||
|
|
||||||
|
logger = LogHelper(logger_name = 'SZ_Import').setup()
|
||||||
|
|
||||||
# 创建配置实例
|
|
||||||
logHelper = LogHelper(
|
|
||||||
level=logging.DEBUG, # 设置日志级别为 DEBUG
|
|
||||||
format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
|
||||||
)
|
|
||||||
|
|
||||||
# # 添加处理器
|
|
||||||
logHelper.add_console_handler() # 默认输出到 stdout
|
|
||||||
logHelper.add_file_handler('Debug.log') # 添加文件日志
|
|
||||||
|
|
||||||
# # 应用配置
|
|
||||||
logHelper.setup()
|
|
||||||
logger = logging.getLogger('StockDataImporter')
|
|
||||||
|
|
||||||
class StockDataImporter:
|
class StockDataImporter:
|
||||||
"""股票数据导入工具(支持新版CSV格式)"""
|
"""股票数据导入工具(支持新版CSV格式)"""
|
||||||
@@ -343,7 +330,7 @@ class StockDataImporter:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
db.execute_update(create_table_sql)
|
db.execute_update(create_table_sql)
|
||||||
logger.info("股票信息表创建成功")
|
logger.info("股票信息表打开成功")
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"创建表失败: {e}")
|
logger.error(f"创建表失败: {e}")
|
||||||
|
|||||||
@@ -1,27 +1,17 @@
|
|||||||
|
"""
|
||||||
|
使用 akshare 数数据源获取/更新A股数据
|
||||||
|
"""
|
||||||
from MySQLHelper import MySQLHelper # 导入我们创建的助手类
|
from MySQLHelper import MySQLHelper # 导入我们创建的助手类
|
||||||
from LogHelper import LogHelper
|
from LogHelper import LogHelper
|
||||||
import logging
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import akshare as ak
|
import akshare as ak
|
||||||
import re
|
import re
|
||||||
import os
|
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from tqdm import tqdm # 用于显示进度条
|
import logging
|
||||||
|
|
||||||
# 创建配置实例
|
# 基本用法(自动创建日期日志+控制台输出)
|
||||||
logHelper = LogHelper(
|
logger = LogHelper(logger_name = 'AkShare').setup()
|
||||||
level=logging.DEBUG, # 设置日志级别为 DEBUG
|
|
||||||
format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
|
||||||
)
|
|
||||||
|
|
||||||
# # 添加处理器
|
|
||||||
logHelper.add_console_handler() # 默认输出到 stdout
|
|
||||||
logHelper.add_file_handler('Debug.log') # 添加文件日志
|
|
||||||
|
|
||||||
# # 应用配置
|
|
||||||
logHelper.setup()
|
|
||||||
logger = logging.getLogger('StockDataImporter')
|
|
||||||
|
|
||||||
# 数据库配置信息 股票列表
|
# 数据库配置信息 股票列表
|
||||||
DB_CONFIG = {
|
DB_CONFIG = {
|
||||||
|
|||||||
Reference in New Issue
Block a user