AMS机器学习课程：预测雷暴旋转的基础机器学习 - 数据

import copy
import warnings
import pathlib
import glob
import typing

import matplotlib.pyplot as pyplot
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn import linear_model

MODULE2_DIR_NAME = '.'
SHORT_COURSE_DIR_NAME = '..'

def feature_file_name_to_date(csv_file_name: str) -> pd.Timestamp:
    file_name = pathlib.Path(csv_file_name).name
    date_string = file_name.replace(
        'track_step_NCARSTORM_d01_', ''
    ).replace('-0000.csv', '')

    return pd.to_datetime(date_string, format='%Y%m%d')

feature_file_name_to_date(
    "../data/track_data_ncar_ams_3km_csv_small/track_step_NCARSTORM_d01_20110308-0000.csv"
)

def find_many_feature_files(
    first_date: str or pd.Timestamp, 
    last_date: str or pd.Timestamp,
    feature_dir_name="../data/track_data_ncar_ams_3km_csv_small"
) -> typing.List:
    first_time = pd.to_datetime(
        first_date
    )
    last_time = pd.to_datetime(
        last_date
    )

    DATE_FORMAT = '%Y%m%d'
    DATE_FORMAT_REGEX = '[0-9][0-9][0-9][0-9][0-1][0-9][0-3][0-9]'
    csv_file_pattern = f'{feature_dir_name}/track_step_NCARSTORM_d01_{DATE_FORMAT_REGEX}-0000.csv'
    csv_file_names = glob.glob(csv_file_pattern)
    csv_file_names.sort()

    file_dates = np.array([
        feature_file_name_to_date(f) for f in csv_file_names
    ])

    good_indices = np.where(np.logical_and(
        file_dates >= first_time,
        file_dates <= last_time
    ))[0]

    return [csv_file_names[k] for k in good_indices]

training_file_names = find_many_feature_files(
    first_date="2010-01-01",
    last_date="2014-12-31",
)
training_file_names[:4]

validation_file_names = find_many_feature_files(
    first_date="20150101",
    last_date="20151231",
)
validation_file_names[-4:]

testing_file_names = find_many_feature_files(
    first_date="20160101",
    last_date="20171231",
)
testing_file_names[:5]

# 元信息列，不需要
METADATA_COLUMNS = [
    'Step_ID', 'Track_ID', 'Ensemble_Name', 'Ensemble_Member', 'Run_Date',
    'Valid_Date', 'Forecast_Hour', 'Valid_Hour_UTC'
]

# 不需要的列
EXTRANEOUS_COLUMNS = [
    'Duration', 'Centroid_Lon', 'Centroid_Lat', 'Centroid_X', 'Centroid_Y',
    'Storm_Motion_U', 'Storm_Motion_V', 'Matched', 'Max_Hail_Size',
    'Num_Matches', 'Shape', 'Location', 'Scale'
]

# 目标列
TARGET_NAME = 'RVORT1_MAX-future_max'

def _remove_future_data(predictor_table):
    predictor_names = list(predictor_table)
    columns_to_remove = [p for p in predictor_names if 'future' in p]

    return predictor_table.drop(
        columns_to_remove, 
        axis=1, 
        inplace=False
    )

def read_feature_file(csv_file_name):
    predictor_table = pd.read_csv(
        csv_file_name, 
        header=0, 
        sep=','
    )
    
    # 删掉不需要的列
    predictor_table.drop(
        EXTRANEOUS_COLUMNS, 
        axis=1, 
        inplace=True
    )
    
    # 元信息表格
    metadata_table = predictor_table[METADATA_COLUMNS]
    predictor_table.drop(
        METADATA_COLUMNS, 
        axis=1, 
        inplace=True
    )

    # 目标表格
    target_table = predictor_table[[TARGET_NAME]]
    predictor_table.drop(
        [TARGET_NAME], 
        axis=1, 
        inplace=True
    )
    
    predictor_table = _remove_future_data(predictor_table)

    return metadata_table, predictor_table, target_table

metadata_table, predictor_table, target_table = read_feature_file(
    "../data/track_data_ncar_ams_3km_csv_small/track_step_NCARSTORM_d01_20110308-0000.csv"
)

metadata_table.head()

predictor_table.head()

target_table.head()

def read_many_feature_files(csv_file_names):
    num_files = len(csv_file_names)
    list_of_metadata_tables = [pd.DataFrame()] * num_files
    list_of_predictor_tables = [pd.DataFrame()] * num_files
    list_of_target_tables = [pd.DataFrame()] * num_files

    for i in tqdm(range(num_files)):
        (
            list_of_metadata_tables[i], 
            list_of_predictor_tables[i],
            list_of_target_tables[i]
        ) = read_feature_file(csv_file_names[i])

        if i == 0:
            continue

        list_of_metadata_tables[i] = list_of_metadata_tables[i].align(
            list_of_metadata_tables[0], 
            axis=1
        )[0]

        list_of_predictor_tables[i] = list_of_predictor_tables[i].align(
            list_of_predictor_tables[0], 
            axis=1
        )[0]

        list_of_target_tables[i] = list_of_target_tables[i].align(
            list_of_target_tables[0], 
            axis=1
        )[0]

    metadata_table = pd.concat(
        list_of_metadata_tables, 
        axis=0, 
        ignore_index=True
    )
    predictor_table = pd.concat(
        list_of_predictor_tables, 
        axis=0, 
        ignore_index=True
    )
    target_table = pd.concat(
        list_of_target_tables, 
        axis=0, 
        ignore_index=True
    )

    return (
        metadata_table, 
        predictor_table, 
        target_table,
    )

(
    training_metadata_table, 
    training_predictor_table_denorm,
    training_target_table
) = read_many_feature_files(training_file_names)

(
    validation_metadata_table, 
    validation_predictor_table_denorm,
    validation_target_table
) = read_many_feature_files(validation_file_names)

(
    testing_metadata_table, 
    testing_predictor_table_denorm,
    testing_target_table
) = read_many_feature_files(testing_file_names)

list(training_metadata_table)

list(training_predictor_table_denorm)

list(training_target_table)

first_predictor_name = list(training_predictor_table_denorm)[0]
first_predictor_name

training_predictor_table_denorm[first_predictor_name][:10]

target_name = list(training_target_table)[0]
target_name

training_target_table[target_name][:10]

def normalize_predictors(
    predictor_table: pd.DataFrame, 
    normalization_dict: typing.Dict or None=None
):
    predictor_names = list(predictor_table)
    num_predictors = len(predictor_names)

    if normalization_dict is None:
        normalization_dict = {}

        for m in range(num_predictors):
            # 计算每列（每个变量）的均值和标准差
            this_mean = np.mean(predictor_table[predictor_names[m]].values)
            this_stdev = np.std(
                predictor_table[predictor_names[m]].values, 
                ddof=1,
            )
            
            # 将计算结果保存到字典对象中
            normalization_dict[predictor_names[m]] = np.array(
                [this_mean, this_stdev]
            )

    for m in range(num_predictors):
        this_mean = normalization_dict[predictor_names[m]][0]
        this_stdev = normalization_dict[predictor_names[m]][1]
        these_norm_values = (
            predictor_table[predictor_names[m]].values - this_mean
        ) / this_stdev
        
        # 使用归一化后的值代替原有数据
        predictor_table = predictor_table.assign(**{
            predictor_names[m]: these_norm_values
        })

    return predictor_table, normalization_dict

def denormalize_predictors(
    predictor_table: pd.DataFrame, 
    normalization_dict: typing.Dict,
):
    predictor_names = list(predictor_table)
    num_predictors = len(predictor_names)

    for m in range(num_predictors):
        this_mean = normalization_dict[predictor_names[m]][0]
        this_stdev = normalization_dict[predictor_names[m]][1]
        these_denorm_values = (
            this_mean + this_stdev * predictor_table[predictor_names[m]].values
        )

        predictor_table = predictor_table.assign(**{
            predictor_names[m]: these_denorm_values
        })

    return predictor_table

predictor_names = list(training_predictor_table_denorm)
training_predictor_table_denorm[predictor_names[0]][:10]

training_predictor_table, normalization_dict = normalize_predictors(
    predictor_table=copy.deepcopy(training_predictor_table_denorm)
)

training_predictor_table[predictor_names[0]][:10]

training_predictor_table_denorm = denormalize_predictors(
    predictor_table=copy.deepcopy(training_predictor_table),
    normalization_dict=normalization_dict
)
training_predictor_table_denorm[predictor_names[0]][:10]

validation_predictor_table, _ = normalize_predictors(
    predictor_table=copy.deepcopy(validation_predictor_table_denorm),
    normalization_dict=normalization_dict,
)

testing_predictor_table, _ = normalize_predictors(
    predictor_table=copy.deepcopy(testing_predictor_table_denorm),
    normalization_dict=normalization_dict,
)