forecast/forecast-algorithm/短期中期部署/部署/OuLiPowerV1.0/model/lightbgm.py

import os
import pandas as pd
import numpy as np
from datetime import datetime, timedelta

import lightgbm as lgb
import joblib
import optuna

from sklearn.utils import resample
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score


# =========================
# 1. 基础配置
# =========================

# 数据文件夹路径
input_folder = '/share/home/jinhm/PhD_Project/A_Data/Data/nwp_processed/OL_JIANGMEN'

# 特征和目标变量
features = ['ssrd', 'fdir', 'tcc', 'lcc', 'mcc', 'hcc', '2t', 'sp', 'tcwv']
target = 'power'

# 每个文件读取前 n 行
n = 288

# 是否对训练集做功率分箱重采样
USE_RESAMPLE = True

# 只建议重采样训练集，不建议重采样验证集和测试集
RESAMPLE_VAL = False

# Optuna 搜索次数
N_TRIALS = 30

# 随机种子
RANDOM_STATE = 42


# =========================
# 2. 生成文件名
# =========================

def gen_file_names(start_date, end_date):
    """
    根据起止日期生成 00 和 12 起报文件路径。
    文件格式示例：2025061400.csv, 2025061412.csv
    """
    files = []
    date_fmt = "%Y%m%d"

    start_dt = datetime.strptime(start_date, date_fmt)
    end_dt = datetime.strptime(end_date, date_fmt)

    current = start_dt
    while current <= end_dt:
        file_00 = os.path.join(input_folder, f"{current.strftime(date_fmt)}00.csv")
        file_12 = os.path.join(input_folder, f"{current.strftime(date_fmt)}12.csv")

        if os.path.exists(file_00):
            files.append(file_00)
        else:
            print(f"文件不存在，跳过: {file_00}")

        if os.path.exists(file_12):
            files.append(file_12)
        else:
            print(f"文件不存在，跳过: {file_12}")

        current += timedelta(days=1)

    return files


# =========================
# 3. 读取数据
# =========================

def load_dataset(start_date, end_date, dataset_name):
    """
    读取指定时间范围内的数据。
    """
    files = gen_file_names(start_date, end_date)
    data_list = []

    print(f"\n开始读取{dataset_name}...")
    print(f"{dataset_name}文件数量: {len(files)}")

    for f in files:
        try:
            df = pd.read_csv(f)
            df_short = df.iloc[:n].copy()

            required_cols = features + [target]
            if all(col in df_short.columns for col in required_cols):
                df_short = df_short[required_cols].copy()

                # 转成数值型，异常值转为 NaN
                for col in required_cols:
                    df_short[col] = pd.to_numeric(df_short[col], errors='coerce')

                # 删除缺失值
                df_short = df_short.dropna(subset=required_cols)

                if len(df_short) > 0:
                    data_list.append(df_short)
            else:
                missing_cols = [col for col in required_cols if col not in df_short.columns]
                print(f"文件缺少字段，跳过: {f}, 缺少: {missing_cols}")

        except Exception as e:
            print(f"Error reading {f}: {e}")

    if not data_list:
        raise ValueError(f"No {dataset_name} data loaded!")

    dataset_df = pd.concat(data_list, axis=0, ignore_index=True)
    print(f"{dataset_name}样本数: {len(dataset_df)}")

    return dataset_df


# =========================
# 4. 功率分箱重采样
# =========================

def resample_power_bins(df, target_col='power', n_bins=10, random_state=42, total_samples=None):
    """
    对大于 0 的功率样本进行分箱重采样，使不同功率区间样本数量更接近。
    零功率样本保留，不参与重采样。

    注意：
    只建议对训练集使用，不建议对验证集和测试集使用。
    """
    df = df.copy()

    # 仅对 power > 0 的样本重采样
    mask = df[target_col] > 0
    df_pos = df.loc[mask].copy()
    df_zero = df.loc[~mask].copy()

    # 如果没有正功率样本，直接返回原始打乱数据
    if len(df_pos) == 0:
        print("警告：没有正功率样本，跳过重采样。")
        return df.sample(frac=1, random_state=random_state).reset_index(drop=True)

    power_vals = df_pos[target_col]

    # 如果所有正功率都一样，无法分箱，直接返回原始打乱数据
    if power_vals.min() == power_vals.max():
        print("警告：正功率样本取值完全相同，跳过重采样。")
        return df.sample(frac=1, random_state=random_state).reset_index(drop=True)

    # 分箱
    bins = np.linspace(power_vals.min(), power_vals.max(), n_bins + 1)
    df_pos.loc[:, 'power_bin'] = pd.cut(power_vals, bins=bins, include_lowest=True)

    # 统计每个 bin 的样本数量
    bin_counts = df_pos['power_bin'].value_counts().sort_index()

    # 以最多的 bin 为基准；也可以指定 total_samples 控制总采样量
    if total_samples is None:
        max_count = int(bin_counts.max())
    else:
        max_count = max(int(total_samples // n_bins), 1)

    resampled_list = []

    for b, group in df_pos.groupby('power_bin', observed=True):
        if len(group) == 0:
            continue

        if len(group) < max_count:
            # 少数区间：有放回上采样
            group_resampled = resample(
                group,
                replace=True,
                n_samples=max_count,
                random_state=random_state
            )
        else:
            # 多数区间：无放回下采样
            group_resampled = resample(
                group,
                replace=False,
                n_samples=max_count,
                random_state=random_state
            )

        resampled_list.append(group_resampled)

    if not resampled_list:
        print("警告：分箱后没有可用样本，跳过重采样。")
        return df.drop(columns=['power_bin'], errors='ignore') \
                 .sample(frac=1, random_state=random_state) \
                 .reset_index(drop=True)

    df_resampled_pos = pd.concat(resampled_list, ignore_index=True)
    df_resampled_pos = df_resampled_pos.drop(columns=['power_bin'], errors='ignore')

    # 合并零功率样本
    df_new = pd.concat([df_zero, df_resampled_pos], ignore_index=True)

    # 打乱
    df_new = df_new.sample(frac=1, random_state=random_state).reset_index(drop=True)

    return df_new


# =========================
# 5. 读取训练、验证、测试集
# =========================

train_df = load_dataset('20250614', '20260228', '训练集')
val_df = load_dataset('20260301', '20260331', '验证集')
test_df = load_dataset('20260401', '20260417', '测试集')


# =========================
# 6. 重采样
# =========================

if USE_RESAMPLE:
    print("\n开始对训练集进行功率分箱重采样...")
    train_df = resample_power_bins(
        train_df,
        target_col=target,
        n_bins=10,
        random_state=RANDOM_STATE
    )
    print(f"重采样后训练集样本数: {len(train_df)}")

    if RESAMPLE_VAL:
        print("\n警告：你开启了验证集重采样，这通常不建议。")
        val_df = resample_power_bins(
            val_df,
            target_col=target,
            n_bins=10,
            random_state=RANDOM_STATE
        )
        print(f"重采样后验证集样本数: {len(val_df)}")
    else:
        print(f"验证集保持原始分布，样本数: {len(val_df)}")
else:
    print("\n未开启重采样。")


# =========================
# 7. 构造 X 和 y
# =========================

X_train = train_df[features]
y_train = train_df[target]

X_val = val_df[features]
y_val = val_df[target]

X_test = test_df[features]
y_test = test_df[target]

print("\n最终数据规模：")
print(f"X_train: {X_train.shape}, y_train: {y_train.shape}")
print(f"X_val:   {X_val.shape}, y_val:   {y_val.shape}")
print(f"X_test:  {X_test.shape}, y_test:  {y_test.shape}")


# =========================
# 8. 评价函数
# =========================

def calc_metrics(y_true, y_pred, name="数据集"):
    """
    计算常用回归指标。
    """
    rmse = np.sqrt(mean_squared_error(y_true, y_pred))
    mae = mean_absolute_error(y_true, y_pred)
    r2 = r2_score(y_true, y_pred)

    print(f"\n{name}评价指标：")
    print(f"{name} RMSE: {rmse:.4f}")
    print(f"{name} MAE : {mae:.4f}")
    print(f"{name} R²  : {r2:.4f}")

    return {
        "rmse": rmse,
        "mae": mae,
        "r2": r2
    }


# =========================
# 9. Optuna 超参数优化
# =========================

def objective(trial):
    """
    Optuna 目标函数：最小化验证集 RMSE。
    """
    params = {
        "objective": "regression",
        "metric": "rmse",

        # 超参数搜索范围
        "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.2, log=True),
        "num_leaves": trial.suggest_int("num_leaves", 16, 128),
        "feature_fraction": trial.suggest_float("feature_fraction", 0.6, 1.0),
        "bagging_fraction": trial.suggest_float("bagging_fraction", 0.6, 1.0),
        "bagging_freq": trial.suggest_int("bagging_freq", 1, 10),
        "min_child_samples": trial.suggest_int("min_child_samples", 5, 50),

        # 固定参数
        "verbose": -1,
        "seed": RANDOM_STATE,
        "feature_fraction_seed": RANDOM_STATE,
        "bagging_seed": RANDOM_STATE,
        "data_random_seed": RANDOM_STATE
    }

    train_set = lgb.Dataset(X_train, y_train)
    val_set = lgb.Dataset(X_val, y_val, reference=train_set)

    gbm = lgb.train(
        params,
        train_set,
        num_boost_round=1000,
        valid_sets=[train_set, val_set],
        valid_names=["train", "valid"],
        callbacks=[
            lgb.early_stopping(stopping_rounds=50, verbose=False),
            lgb.log_evaluation(period=0)
        ]
    )

    val_pred = gbm.predict(X_val, num_iteration=gbm.best_iteration)
    val_rmse = np.sqrt(mean_squared_error(y_val, val_pred))

    return val_rmse


print("\n开始使用 Optuna 进行超参数优化...")

study = optuna.create_study(
    direction="minimize",
    sampler=optuna.samplers.TPESampler(seed=RANDOM_STATE)
)

study.optimize(objective, n_trials=N_TRIALS)

print("\nOptuna 优化完成。")
print("最优 RMSE:", study.best_value)
print("最优超参数:")
for k, v in study.best_params.items():
    print(f"  {k}: {v}")


# =========================
# 10. 使用最优参数训练最终模型
# =========================

best_params = study.best_params.copy()

best_params.update({
    "objective": "regression",
    "metric": "rmse",
    "verbose": -1,
    "seed": RANDOM_STATE,
    "feature_fraction_seed": RANDOM_STATE,
    "bagging_seed": RANDOM_STATE,
    "data_random_seed": RANDOM_STATE
})

print("\n用最优参数训练最终 LightGBM 模型...")

train_set = lgb.Dataset(X_train, y_train)
val_set = lgb.Dataset(X_val, y_val, reference=train_set)

gbm = lgb.train(
    best_params,
    train_set,
    num_boost_round=1000,
    valid_sets=[train_set, val_set],
    valid_names=["train", "valid"],
    callbacks=[
        lgb.early_stopping(stopping_rounds=50, verbose=True),
        lgb.log_evaluation(period=100)
    ]
)

print("\n训练完毕。")
print(f"最佳迭代轮数 best_iteration: {gbm.best_iteration}")


# =========================
# 11. 模型评估
# =========================

val_pred = gbm.predict(X_val, num_iteration=gbm.best_iteration)
test_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)

val_metrics = calc_metrics(y_val, val_pred, name="验证集")
test_metrics = calc_metrics(y_test, test_pred, name="测试集")


# =========================
# 12. 保存模型
# =========================

model_save_path_txt = './lightgbm_power_model.txt'
model_save_path_pkl = './lightgbm_power_model.pkl'

gbm.save_model(model_save_path_txt)
joblib.dump(gbm, model_save_path_pkl)

print("\n模型已保存：")
print(f"TXT模型: {model_save_path_txt}")
print(f"PKL模型: {model_save_path_pkl}")


# =========================
# 13. 保存预测结果
# =========================

val_result = pd.DataFrame({
    "y_true": y_val.values,
    "y_pred": val_pred
})

test_result = pd.DataFrame({
    "y_true": y_test.values,
    "y_pred": test_pred
})

val_result.to_csv("./lightgbm_val_prediction.csv", index=False)
test_result.to_csv("./lightgbm_test_prediction.csv", index=False)

print("\n预测结果已保存：")
print("验证集预测结果: ./lightgbm_val_prediction.csv")
print("测试集预测结果: ./lightgbm_test_prediction.csv")