update flies
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -264,6 +264,4 @@ compile_commands.json
|
||||
|
||||
*_qmlcache.qrc
|
||||
|
||||
|
||||
data/
|
||||
log/
|
||||
logs/
|
||||
|
||||
97
LogHelper.py
97
LogHelper.py
@@ -1,63 +1,90 @@
|
||||
"""
|
||||
日志管理类
|
||||
|
||||
日志输出到运行目录指定文件夹
|
||||
日志文件按照:YYYY-MM-dd.log 格式输出
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
class LogHelper:
|
||||
# 内部日志级别映射表
|
||||
_LEVEL_MAP = {
|
||||
'DEBUG': 10,
|
||||
'INFO': 20,
|
||||
'WARNING': 30,
|
||||
'ERROR': 40,
|
||||
'CRITICAL': 50
|
||||
}
|
||||
|
||||
def __init__(self,
|
||||
level=logging.INFO,
|
||||
level='INFO',
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=None):
|
||||
handlers=None,
|
||||
log_dir="logs",
|
||||
logger_name=None):
|
||||
"""
|
||||
初始化日志配置
|
||||
|
||||
:param level: 日志级别,默认为 logging.INFO
|
||||
:param format: 日志格式字符串
|
||||
:param handlers: 日志处理器列表,默认为空(会自动添加控制台处理器)
|
||||
:param log_dir: 日志存储目录,默认为"logs"
|
||||
"""
|
||||
self.level = level
|
||||
self.level = self._LEVEL_MAP.get(level.upper(), 20) # 默认INFO
|
||||
self.format = format
|
||||
self.handlers = handlers if handlers is not None else []
|
||||
self.log_dir = log_dir
|
||||
self.logger_name = logger_name
|
||||
# 确保日志目录存在
|
||||
os.makedirs(self.log_dir, exist_ok=True)
|
||||
|
||||
def add_console_handler(self, stream=sys.stdout, encoding='utf-8'):
|
||||
"""添加控制台处理器"""
|
||||
console_handler = logging.StreamHandler(stream)
|
||||
console_handler.setFormatter(logging.Formatter(self.format))
|
||||
# 设置控制台编码
|
||||
if encoding:
|
||||
console_handler.encoding = encoding
|
||||
self.handlers.append(console_handler)
|
||||
return self # 支持链式调用
|
||||
|
||||
def add_file_handler(self, filename, encoding='utf-8'):
|
||||
"""添加文件处理器(解决中文乱码问题)"""
|
||||
# 使用支持UTF-8编码的文件处理器
|
||||
filepath = "log/" + filename
|
||||
file_handler = logging.FileHandler(filepath, encoding=encoding)
|
||||
def _get_daily_log_path(self):
|
||||
"""生成基于当前日期的日志文件路径"""
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
filename = f"{today}.log"
|
||||
return os.path.join(self.log_dir, filename)
|
||||
|
||||
def _add_daily_file_handler(self, encoding='utf-8'):
|
||||
"""添加按日期自动创建的文件处理器"""
|
||||
log_path = self._get_daily_log_path()
|
||||
file_handler = logging.FileHandler(log_path, encoding=encoding)
|
||||
file_handler.setFormatter(logging.Formatter(self.format))
|
||||
self.handlers.append(file_handler)
|
||||
return self # 支持链式调用
|
||||
|
||||
def setup(self):
|
||||
"""应用日志配置"""
|
||||
logging.basicConfig(
|
||||
level=self.level,
|
||||
format=self.format,
|
||||
handlers=self.handlers
|
||||
)
|
||||
|
||||
# # 使用示例
|
||||
# if __name__ == "__main__":
|
||||
# # 创建配置实例
|
||||
# logger_config = LoggerConfig(
|
||||
# level=logging.DEBUG, # 设置日志级别为 DEBUG
|
||||
# format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
||||
# )
|
||||
|
||||
# # # 添加处理器
|
||||
# logger_config.add_console_handler() # 默认输出到 stdout
|
||||
# logger_config.add_file_handler('Debug.log') # 添加文件日志
|
||||
|
||||
# # # 应用配置
|
||||
# logger_config.setup()
|
||||
|
||||
# # 测试日志
|
||||
# logging.debug("Debug 信息")
|
||||
# logging.info("Info 信息")
|
||||
# logging.warning("警告信息")
|
||||
"""应用日志配置(自动添加日期文件处理器)"""
|
||||
# 如果未提供任何处理器,默认添加控制台和日期文件处理器
|
||||
if not self.handlers:
|
||||
self.add_console_handler()
|
||||
self._add_daily_file_handler()
|
||||
|
||||
# 获取或创建logger
|
||||
logger = logging.getLogger(self.logger_name)
|
||||
logger.setLevel(self.level)
|
||||
|
||||
# 移除所有现有处理器(避免重复添加)
|
||||
for handler in logger.handlers[:]:
|
||||
logger.removeHandler(handler)
|
||||
|
||||
# 添加配置的处理器
|
||||
for handler in self.handlers:
|
||||
logger.addHandler(handler)
|
||||
|
||||
# 确保日志消息不会传递给父logger(避免重复记录)
|
||||
logger.propagate = False
|
||||
|
||||
return logger
|
||||
@@ -3,27 +3,24 @@
|
||||
—— 增加事务管理
|
||||
—— 增加ID获取
|
||||
—— 增加表操作等使用功能
|
||||
—— 数据使用过程中出现异常时,才输出日志
|
||||
"""
|
||||
import pymysql
|
||||
from pymysql import Error
|
||||
from typing import List, Dict, Union, Optional, Tuple
|
||||
from contextlib import contextmanager
|
||||
from LogHelper import LogHelper
|
||||
import logging
|
||||
|
||||
# 创建配置实例
|
||||
logHelper = LogHelper(
|
||||
level=logging.DEBUG, # 设置日志级别为 DEBUG
|
||||
format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
||||
)
|
||||
# 基本用法(自动创建日期日志+控制台输出)
|
||||
logger = LogHelper(logger_name = 'database').setup()
|
||||
|
||||
# # 添加处理器
|
||||
logHelper.add_console_handler() # 默认输出到 stdout
|
||||
logHelper.add_file_handler('Debug.log') # 添加文件日志
|
||||
# # 高级用法(自定义配置)
|
||||
# logger = LogHelper(
|
||||
# level=logging.DEBUG,
|
||||
# log_dir="databaselogs",
|
||||
# format='%(levelname)s - %(message)s'
|
||||
# ).setup()
|
||||
|
||||
# # 应用配置
|
||||
logHelper.setup()
|
||||
logger = logging.getLogger('StockDataImporter')
|
||||
|
||||
class MySQLHelper:
|
||||
def __init__(self, host: str, user: str, password: str, database: str,
|
||||
@@ -62,7 +59,7 @@ class MySQLHelper:
|
||||
cursorclass=pymysql.cursors.DictCursor # 返回字典形式的结果
|
||||
)
|
||||
self.cursor = self.connection.cursor()
|
||||
logger.info("MySQL数据库连接成功")
|
||||
# logger.info("MySQL数据库连接成功")
|
||||
return True
|
||||
except Error as e:
|
||||
logger.error(f"连接MySQL数据库失败: {e}")
|
||||
@@ -76,7 +73,7 @@ class MySQLHelper:
|
||||
self.cursor.close()
|
||||
if self.connection:
|
||||
self.connection.close()
|
||||
logger.info("MySQL数据库连接已关闭")
|
||||
# logger.info("MySQL数据库连接已关闭")
|
||||
|
||||
def execute_query(self, sql: str, params: Union[Tuple, List, Dict, None] = None) -> List[Dict]:
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
"""
|
||||
使用 baostock 数数据源获取/更新A股数据
|
||||
|
||||
***
|
||||
该数据源中会返回非交易日数据,导致数据存储异常
|
||||
"""
|
||||
from MySQLHelper import MySQLHelper
|
||||
from LogHelper import LogHelper
|
||||
import logging
|
||||
import pandas as pd
|
||||
import re
|
||||
import time
|
||||
@@ -8,16 +13,8 @@ from datetime import datetime, timedelta
|
||||
from tqdm import tqdm
|
||||
import baostock as bs
|
||||
|
||||
# 创建配置实例
|
||||
logHelper = LogHelper(
|
||||
level=logging.DEBUG,
|
||||
format='%(asctime)s [%(levelname)s] %(message)s'
|
||||
)
|
||||
|
||||
logHelper.add_console_handler()
|
||||
logHelper.add_file_handler('Debug.log')
|
||||
logHelper.setup()
|
||||
logger = logging.getLogger('StockDataImporter')
|
||||
# 基本用法(自动创建日期日志+控制台输出)
|
||||
logger = LogHelper(logger_name = 'baoStock').setup()
|
||||
|
||||
# 数据库配置
|
||||
DB_CONFIG = {
|
||||
@@ -100,7 +97,7 @@ def create_stock_table(db: MySQLHelper, table_name: str) -> bool:
|
||||
|
||||
try:
|
||||
db.execute_update(create_table_sql)
|
||||
logger.info(f"成功创建表: {table_name}")
|
||||
logger.info(f"成功打开表: {table_name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"创建表 {table_name} 失败: {e}")
|
||||
|
||||
2871
data/A股列表.csv
Normal file
2871
data/A股列表.csv
Normal file
File diff suppressed because it is too large
Load Diff
BIN
data/A股列表.xlsx
Normal file
BIN
data/A股列表.xlsx
Normal file
Binary file not shown.
1695
data/GPLIST.csv
Normal file
1695
data/GPLIST.csv
Normal file
File diff suppressed because it is too large
Load Diff
BIN
data/GPLIST.xlsx
Normal file
BIN
data/GPLIST.xlsx
Normal file
Binary file not shown.
@@ -8,7 +8,6 @@
|
||||
|
||||
import pandas as pd
|
||||
import os
|
||||
import logging
|
||||
import sys
|
||||
import csv
|
||||
import chardet # 用于检测文件编码
|
||||
@@ -18,19 +17,7 @@ from datetime import datetime
|
||||
from MySQLHelper import MySQLHelper
|
||||
from LogHelper import LogHelper
|
||||
|
||||
# 创建配置实例
|
||||
logHelper = LogHelper(
|
||||
level=logging.DEBUG, # 设置日志级别为 DEBUG
|
||||
format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
||||
)
|
||||
|
||||
# # 添加处理器
|
||||
logHelper.add_console_handler() # 默认输出到 stdout
|
||||
logHelper.add_file_handler('Debug.log') # 添加文件日志
|
||||
|
||||
# # 应用配置
|
||||
logHelper.setup()
|
||||
logger = logging.getLogger('StockDataImporter')
|
||||
logger = LogHelper(logger_name = 'SH_Import').setup()
|
||||
|
||||
class StockDataImporter:
|
||||
"""股票数据导入工具(支持CSV)"""
|
||||
|
||||
@@ -11,25 +11,12 @@ from MySQLHelper import MySQLHelper
|
||||
from LogHelper import LogHelper
|
||||
import pandas as pd
|
||||
import os
|
||||
import logging
|
||||
import sys
|
||||
import csv
|
||||
import chardet
|
||||
|
||||
logger = LogHelper(logger_name = 'SZ_Import').setup()
|
||||
|
||||
# 创建配置实例
|
||||
logHelper = LogHelper(
|
||||
level=logging.DEBUG, # 设置日志级别为 DEBUG
|
||||
format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
||||
)
|
||||
|
||||
# # 添加处理器
|
||||
logHelper.add_console_handler() # 默认输出到 stdout
|
||||
logHelper.add_file_handler('Debug.log') # 添加文件日志
|
||||
|
||||
# # 应用配置
|
||||
logHelper.setup()
|
||||
logger = logging.getLogger('StockDataImporter')
|
||||
|
||||
class StockDataImporter:
|
||||
"""股票数据导入工具(支持新版CSV格式)"""
|
||||
@@ -343,7 +330,7 @@ class StockDataImporter:
|
||||
|
||||
try:
|
||||
db.execute_update(create_table_sql)
|
||||
logger.info("股票信息表创建成功")
|
||||
logger.info("股票信息表打开成功")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"创建表失败: {e}")
|
||||
|
||||
@@ -1,27 +1,17 @@
|
||||
"""
|
||||
使用 akshare 数数据源获取/更新A股数据
|
||||
"""
|
||||
from MySQLHelper import MySQLHelper # 导入我们创建的助手类
|
||||
from LogHelper import LogHelper
|
||||
import logging
|
||||
import pandas as pd
|
||||
import akshare as ak
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from tqdm import tqdm # 用于显示进度条
|
||||
import logging
|
||||
|
||||
# 创建配置实例
|
||||
logHelper = LogHelper(
|
||||
level=logging.DEBUG, # 设置日志级别为 DEBUG
|
||||
format='%(asctime)s [%(levelname)s] %(message)s' # 自定义格式
|
||||
)
|
||||
|
||||
# # 添加处理器
|
||||
logHelper.add_console_handler() # 默认输出到 stdout
|
||||
logHelper.add_file_handler('Debug.log') # 添加文件日志
|
||||
|
||||
# # 应用配置
|
||||
logHelper.setup()
|
||||
logger = logging.getLogger('StockDataImporter')
|
||||
# 基本用法(自动创建日期日志+控制台输出)
|
||||
logger = LogHelper(logger_name = 'AkShare').setup()
|
||||
|
||||
# 数据库配置信息 股票列表
|
||||
DB_CONFIG = {
|
||||
|
||||
Reference in New Issue
Block a user