181 lines
7.1 KiB
Python
181 lines
7.1 KiB
Python
|
import gc
|
|||
|
|
import os
|
|||
|
|
|
|||
|
|
import numpy as np
|
|||
|
|
import pandas as pd
|
|||
|
|
import matplotlib.pyplot as plt
|
|||
|
|
from scipy import stats
|
|||
|
|
|
|||
|
|
|
|||
|
|
def generate_statistical_overview(self):
|
|||
|
|
"""生成统计概览 - 优化内存版本"""
|
|||
|
|
fig = None
|
|||
|
|
try:
|
|||
|
|
self._log_step("Generating statistical overview...")
|
|||
|
|
|
|||
|
|
# 检查数据
|
|||
|
|
if not hasattr(self, 'data') or self.data is None or len(self.data) == 0:
|
|||
|
|
self._log_step("No data available for statistical overview", "warning")
|
|||
|
|
return None, "No data available", None
|
|||
|
|
|
|||
|
|
# 计算统计数据
|
|||
|
|
numeric_cols = self.data.select_dtypes(include=[np.number]).columns
|
|||
|
|
stats_df = self.data[numeric_cols].describe().T.reset_index().rename(columns={'index': 'variable'})
|
|||
|
|
summary = f"Generated statistical overview for {len(numeric_cols)} variables"
|
|||
|
|
|
|||
|
|
if not self.generate_plots:
|
|||
|
|
self._log_step("Statistical overview generated (data only)", "success")
|
|||
|
|
return None, summary, stats_df
|
|||
|
|
|
|||
|
|
# 使用更小的图形尺寸和DPI来节省内存
|
|||
|
|
fig, axes = plt.subplots(2, 2, figsize=(10, 8), dpi=100)
|
|||
|
|
fig.suptitle('Statistical Overview', fontsize=14)
|
|||
|
|
|
|||
|
|
# 基本统计信息
|
|||
|
|
# 只处理前4个变量以节省内存
|
|||
|
|
num_vars = min(4, len(self.data.columns))
|
|||
|
|
|
|||
|
|
for i in range(num_vars):
|
|||
|
|
row = i // 2
|
|||
|
|
col = i % 2
|
|||
|
|
col_name = self.data.columns[i]
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 时间序列图
|
|||
|
|
axes[row, col].plot(self.data.index, self.data[col_name], linewidth=1)
|
|||
|
|
axes[row, col].set_title(f'{col_name}')
|
|||
|
|
axes[row, col].tick_params(axis='x', rotation=45)
|
|||
|
|
axes[row, col].grid(True, alpha=0.3)
|
|||
|
|
except Exception as e:
|
|||
|
|
self._log_step(f"Plotting {col_name} failed: {e}", "warning")
|
|||
|
|
axes[row, col].text(
|
|||
|
|
0.5,
|
|||
|
|
0.5,
|
|||
|
|
f'Error: {str(e)[:30]}',
|
|||
|
|
ha='center',
|
|||
|
|
va='center',
|
|||
|
|
transform=axes[row, col].transAxes,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
plt.tight_layout()
|
|||
|
|
|
|||
|
|
# 保存图片（使用更低的DPI）
|
|||
|
|
img_path = os.path.join(self.temp_dir.name, 'stats_overview.png')
|
|||
|
|
try:
|
|||
|
|
plt.savefig(img_path, dpi=100, bbox_inches='tight', format='png')
|
|||
|
|
if not os.path.exists(img_path):
|
|||
|
|
self._log_step("Failed to save statistical overview image", "error")
|
|||
|
|
return None, "Failed to save image", stats_df
|
|||
|
|
except Exception as save_error:
|
|||
|
|
self._log_step(f"Failed to save figure: {save_error}", "error")
|
|||
|
|
return None, f"Save error: {str(save_error)[:100]}", stats_df
|
|||
|
|
finally:
|
|||
|
|
plt.close(fig) # 明确关闭图形释放内存
|
|||
|
|
gc.collect()
|
|||
|
|
|
|||
|
|
self._log_step("Statistical overview generated", "success")
|
|||
|
|
|
|||
|
|
return img_path, summary, stats_df
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
self._log_step(f"Statistical overview failed: {str(e)[:100]}", "error")
|
|||
|
|
if fig is not None:
|
|||
|
|
try:
|
|||
|
|
plt.close(fig)
|
|||
|
|
gc.collect()
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
return None, f"Statistical overview failed: {str(e)[:100]}", None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def perform_normality_tests(self):
|
|||
|
|
"""执行正态性检验"""
|
|||
|
|
try:
|
|||
|
|
self._log_step("Performing normality tests...")
|
|||
|
|
|
|||
|
|
if hasattr(self, 'data') and self.data is not None:
|
|||
|
|
numeric_cols = self.data.select_dtypes(include=[np.number]).columns
|
|||
|
|
results = {}
|
|||
|
|
|
|||
|
|
for col in numeric_cols[:3]: # 只测试前3个变量
|
|||
|
|
series = self.data[col].dropna()
|
|||
|
|
col_results = {}
|
|||
|
|
|
|||
|
|
# 直方图分箱（后端负责 binning）
|
|||
|
|
hist_counts, bin_edges = np.histogram(series, bins=20)
|
|||
|
|
histogram = []
|
|||
|
|
for i in range(len(hist_counts)):
|
|||
|
|
histogram.append({
|
|||
|
|
'range_start': float(bin_edges[i]),
|
|||
|
|
'range_end': float(bin_edges[i + 1]),
|
|||
|
|
'count': int(hist_counts[i])
|
|||
|
|
})
|
|||
|
|
col_results['histogram'] = histogram
|
|||
|
|
|
|||
|
|
# Shapiro-Wilk检验
|
|||
|
|
if len(series) >= 3 and len(series) <= 5000:
|
|||
|
|
shapiro_result = stats.shapiro(series)
|
|||
|
|
col_results['Shapiro-Wilk'] = {
|
|||
|
|
'statistic': float(shapiro_result[0]),
|
|||
|
|
'p_value': float(shapiro_result[1]),
|
|||
|
|
'normal': bool(shapiro_result[1] > 0.05),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Jarque-Bera检验
|
|||
|
|
jb_result = stats.jarque_bera(series)
|
|||
|
|
# SciPy result typing varies by version; keep runtime behavior and silence stub mismatch.
|
|||
|
|
jb_stat = float(jb_result[0]) # type: ignore[index,arg-type]
|
|||
|
|
jb_p = float(jb_result[1]) # type: ignore[index,arg-type]
|
|||
|
|
col_results['Jarque-Bera'] = {
|
|||
|
|
'statistic': jb_stat,
|
|||
|
|
'p_value': jb_p,
|
|||
|
|
'normal': bool(jb_p > 0.05),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
results[col] = col_results
|
|||
|
|
|
|||
|
|
summary = f"正态性检验完成，测试了 {len(results)} 个变量"
|
|||
|
|
|
|||
|
|
if not self.generate_plots:
|
|||
|
|
self._log_step("Normality tests completed (data only)", "success")
|
|||
|
|
return None, summary, results
|
|||
|
|
|
|||
|
|
# 创建正态性检验可视化
|
|||
|
|
n_cols = min(3, len(numeric_cols))
|
|||
|
|
fig, axes = plt.subplots(n_cols, 2, figsize=(12, 4 * n_cols))
|
|||
|
|
fig.suptitle('正态性检验结果', fontsize=16)
|
|||
|
|
|
|||
|
|
if n_cols == 1:
|
|||
|
|
axes = axes.reshape(1, -1)
|
|||
|
|
|
|||
|
|
for i, col in enumerate(numeric_cols[:n_cols]):
|
|||
|
|
series = self.data[col].dropna()
|
|||
|
|
|
|||
|
|
# 直方图与正态曲线
|
|||
|
|
axes[i, 0].hist(series, bins=20, density=True, alpha=0.7, color='skyblue')
|
|||
|
|
xmin, xmax = axes[i, 0].get_xlim()
|
|||
|
|
x = np.linspace(xmin, xmax, 100)
|
|||
|
|
p = stats.norm.pdf(x, series.mean(), series.std())
|
|||
|
|
axes[i, 0].plot(x, p, 'k', linewidth=2)
|
|||
|
|
axes[i, 0].set_title(f'{col} - 分布直方图')
|
|||
|
|
|
|||
|
|
# Q-Q图
|
|||
|
|
stats.probplot(series, dist="norm", plot=axes[i, 1])
|
|||
|
|
axes[i, 1].set_title(f'{col} - Q-Q图')
|
|||
|
|
|
|||
|
|
plt.tight_layout()
|
|||
|
|
img_path = os.path.join(self.temp_dir.name, 'normality_tests.png')
|
|||
|
|
plt.savefig(img_path, dpi=150, bbox_inches='tight')
|
|||
|
|
plt.close()
|
|||
|
|
|
|||
|
|
self._log_step("Normality tests completed", "success")
|
|||
|
|
|
|||
|
|
return img_path, summary, results
|
|||
|
|
|
|||
|
|
self._log_step("No data available for normality tests", "warning")
|
|||
|
|
return None, "数据不足，无法进行正态性检验", None
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
self._log_step(f"Normality tests failed: {e}", "error")
|
|||
|
|
return None, f"正态性检验失败: {e}", None
|