修改排序方式和更新内容,
This commit is contained in:
@@ -9,26 +9,51 @@ import tushare as ts
|
||||
import time
|
||||
from functools import lru_cache
|
||||
|
||||
# -------------------------- 配置参数区域 --------------------------
|
||||
class Config:
|
||||
"""配置参数类,集中管理所有配置"""
|
||||
# 日志配置
|
||||
LOG_NAME = 'market_data_check'
|
||||
LOG_FILE = 'market_data_check.log'
|
||||
LOG_LEVEL = logging.INFO
|
||||
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
|
||||
LOG_CONSOLE = True
|
||||
LOG_DIR = '.'
|
||||
LOG_BACKUP_COUNT = 3
|
||||
|
||||
# Tushare配置
|
||||
TUSHARE_TOKENS = [
|
||||
'9343e641869058684afeadfcfe7fd6684160852e52e85332a7734c8d' # 主账户
|
||||
]
|
||||
|
||||
# API请求频率控制
|
||||
MAX_REQUESTS_PER_MINUTE = 500
|
||||
|
||||
# 数据目录配置
|
||||
DATA_DIR = Path(r'D:\gp_data\day')
|
||||
|
||||
# 交易日历配置
|
||||
TRADE_CALENDAR_START_YEARS = 2 # 过去2年
|
||||
TRADE_CALENDAR_END_MONTHS = 1 # 未来1个月
|
||||
|
||||
# 输出文件配置
|
||||
OUTPUT_FILE = Path('market_data_check_result.csv')
|
||||
|
||||
# 默认参数配置
|
||||
DEFAULT_ONLINE_CHECK = False
|
||||
|
||||
# 配置日志
|
||||
logger = setup_logger(
|
||||
name='market_data_check',
|
||||
log_file='market_data_check.log',
|
||||
level=logging.INFO,
|
||||
log_format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
console=True,
|
||||
log_dir='.',
|
||||
backup_count=3
|
||||
name=Config.LOG_NAME,
|
||||
log_file=Config.LOG_FILE,
|
||||
level=Config.LOG_LEVEL,
|
||||
log_format=Config.LOG_FORMAT,
|
||||
console=Config.LOG_CONSOLE,
|
||||
log_dir=Config.LOG_DIR,
|
||||
backup_count=Config.LOG_BACKUP_COUNT
|
||||
)
|
||||
logger.propagate = False # 避免日志消息向上传递到父记录器,防止重复输出
|
||||
|
||||
# Tushare配置
|
||||
TUSHARE_TOKENS = [
|
||||
'9343e641869058684afeadfcfe7fd6684160852e52e85332a7734c8d' # 主账户
|
||||
]
|
||||
|
||||
# API请求频率控制
|
||||
MAX_REQUESTS_PER_MINUTE = 500
|
||||
|
||||
class TushareManager:
|
||||
"""Tushare API管理类,处理账户轮询和请求频率控制"""
|
||||
def __init__(self, tokens):
|
||||
@@ -55,7 +80,7 @@ class TushareManager:
|
||||
self.last_request_time = current_time
|
||||
|
||||
# 如果请求次数超过限制,等待
|
||||
if self.request_count >= MAX_REQUESTS_PER_MINUTE:
|
||||
if self.request_count >= Config.MAX_REQUESTS_PER_MINUTE:
|
||||
wait_time = 60 - time_since_last_request + 1
|
||||
logger.info(f"请求频率过高,等待 {wait_time:.1f} 秒")
|
||||
time.sleep(wait_time)
|
||||
@@ -70,7 +95,7 @@ class TushareManager:
|
||||
self.last_request_time = current_time
|
||||
|
||||
# 创建Tushare管理器实例
|
||||
tushare_manager = TushareManager(TUSHARE_TOKENS)
|
||||
tushare_manager = TushareManager(Config.TUSHARE_TOKENS)
|
||||
|
||||
# 全局变量,用于缓存交易日历
|
||||
trade_calendar_cache = None
|
||||
@@ -88,10 +113,10 @@ def get_trade_calendar():
|
||||
return trade_calendar_cache
|
||||
|
||||
try:
|
||||
# 计算日期范围:过去2年到未来1个月
|
||||
# 计算日期范围:过去N年到未来M个月
|
||||
today = datetime.datetime.now()
|
||||
start_date = (today - datetime.timedelta(days=730)).strftime('%Y%m%d') # 过去2年
|
||||
end_date = (today + datetime.timedelta(days=30)).strftime('%Y%m%d') # 未来1个月
|
||||
start_date = (today - datetime.timedelta(days=365 * Config.TRADE_CALENDAR_START_YEARS)).strftime('%Y%m%d')
|
||||
end_date = (today + datetime.timedelta(days=30 * Config.TRADE_CALENDAR_END_MONTHS)).strftime('%Y%m%d')
|
||||
|
||||
pro = tushare_manager.get_pro_api()
|
||||
tushare_manager.control_request_rate()
|
||||
@@ -116,6 +141,8 @@ def get_trade_calendar():
|
||||
def get_latest_trade_date(file_path):
|
||||
"""
|
||||
从txt文件中获取最新的交易日期
|
||||
注意:现在数据文件按日期降序保存,最新的交易日期在文件第一行数据(跳过表头)
|
||||
|
||||
:param file_path: 文件路径
|
||||
:return: 最新交易日期字符串,如'20251204',如果文件为空返回None
|
||||
"""
|
||||
@@ -125,17 +152,27 @@ def get_latest_trade_date(file_path):
|
||||
if len(lines) < 2: # 至少需要有表头和一行数据
|
||||
logger.warning(f"文件 {file_path} 内容不足")
|
||||
return None
|
||||
# 第二行是第一行数据(最新的交易日期)
|
||||
first_data_line = lines[1].strip()
|
||||
if not first_data_line:
|
||||
logger.warning(f"文件 {file_path} 数据行为空")
|
||||
|
||||
# 解析表头,找到trade_date列的索引
|
||||
header_line = lines[0].strip()
|
||||
headers = header_line.split('\t')
|
||||
if 'trade_date' not in headers:
|
||||
logger.warning(f"文件 {file_path} 缺少trade_date列")
|
||||
return None
|
||||
# 按制表符分割
|
||||
columns = first_data_line.split('\t')
|
||||
if len(columns) < 2: # 至少需要有ts_code和trade_date
|
||||
logger.warning(f"文件 {file_path} 数据格式不正确")
|
||||
return None
|
||||
return columns[1]
|
||||
trade_date_idx = headers.index('trade_date')
|
||||
|
||||
# 跳过空行,从文件开头查找第一行有效数据(最新日期)
|
||||
for i in range(1, len(lines)): # 从第二行开始查找,跳过表头行(0)
|
||||
line = lines[i].strip()
|
||||
if line: # 找到非空行
|
||||
columns = line.split('\t')
|
||||
if len(columns) <= trade_date_idx: # 确保有足够的列
|
||||
logger.warning(f"文件 {file_path} 数据格式不正确")
|
||||
return None
|
||||
return columns[trade_date_idx]
|
||||
# 如果没有找到有效数据行
|
||||
logger.warning(f"文件 {file_path} 无有效数据行")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"读取文件 {file_path} 时出错: {str(e)}")
|
||||
return None
|
||||
@@ -199,7 +236,7 @@ def check_online_data_exists(ts_code, trade_date):
|
||||
logger.error(f"查询在线数据失败 {ts_code} {trade_date}: {str(e)}")
|
||||
return None
|
||||
|
||||
def check_market_data(online_check=False):
|
||||
def check_market_data(online_check=Config.DEFAULT_ONLINE_CHECK):
|
||||
"""
|
||||
检查所有行情数据文件的完整性
|
||||
|
||||
@@ -207,7 +244,7 @@ def check_market_data(online_check=False):
|
||||
online_check: 是否进行在线数据检查,默认False
|
||||
"""
|
||||
# 设置数据目录
|
||||
data_dir = Path(r'D:\gp_data\day')
|
||||
data_dir = Config.DATA_DIR
|
||||
|
||||
# 获取当天日期(格式:YYYYMMDD)
|
||||
today = datetime.datetime.now().strftime('%Y%m%d')
|
||||
@@ -278,7 +315,7 @@ def check_market_data(online_check=False):
|
||||
print()
|
||||
|
||||
# 输出结果到CSV文件
|
||||
output_file = Path('market_data_check_result.csv')
|
||||
output_file = Config.OUTPUT_FILE
|
||||
with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
|
||||
fieldnames = ['file_name', 'ts_code', 'latest_date', 'trading_days_diff', 'online_data_exists', 'status']
|
||||
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
||||
|
||||
Reference in New Issue
Block a user