import pandas as pd#导入csv文件的库
import numpy as np#进行矩阵运算的库
import polars as pl#和pandas类似,但是处理大型数据集有更好的性能.
#用于对一组元素计数,一个存在默认值的字典,访问不存在的值时抛出的是默认值
from collections import Counter,defaultdict
import re#用于正则表达式提取
from scipy.stats import skew, kurtosis#统计分析和概率分布导入偏度和峰度

#model
from lightgbm import LGBMRegressor#导入lgbm回归器
from catboost import CatBoostRegressor#catboost回归器
from xgboost import XGBRegressor#导入XGB回归器

#KFold是直接分成k折,StratifiedKFold还要考虑每种类别的占比
from sklearn.model_selection import StratifiedKFold

#设置随机种子,保证模型可以复现
import random
seed=2024
np.random.seed(seed)
random.seed(seed)

import warnings#避免一些可以忽略的报错
warnings.filterwarnings('ignore')#filterwarnings()方法是用于设置警告过滤器的方法，它可以控制警告信息的输出方式和级别。

def get_Essays(df):
    #传入的是其中一个id的论文,取出id
    USER_ID = df["id"].iloc[0]
    #这个id和论文内容相关的几列
    textInputDf = df[['activity', 'cursor_position', 'text_change']]
    #对论文没有变化的动作不用管
    currTextInput = textInputDf[textInputDf.activity != 'Nonproduction']
    essayText = ""#开始重构论文内容.
    for Input in currTextInput.values:#取出一个'activity', 'cursor_position', 'text_change'
        #input[0]是这个id的activity
        if Input[0] == 'Replace':
            #text_change按照' => '分开 replaceTxt:[' qqq qqqqq ', ' ']
            replaceTxt = Input[2].split(' => ')#应该是A=>B的操作
            #input[1]是鼠标位置,是一个数字 鼠标位置-len()
            #这是一个字符串的转换操作,由replaceTxt[0]转成replaceTxt[1] 
            essayText = essayText[:Input[1] - len(replaceTxt[1])] + replaceTxt[1] +essayText[Input[1] - len(replaceTxt[1]) + len(replaceTxt[0]):]
            continue
        if Input[0] == 'Paste':#在Input[1]的位置粘贴Input[2]
            essayText = essayText[:Input[1] - len(Input[2])] + Input[2] + essayText[Input[1] - len(Input[2]):]
            continue
        if Input[0] == 'Remove/Cut':#删除剪切 在Input[1]的位置删除Input[2]
            essayText = essayText[:Input[1]] + essayText[Input[1] + len(Input[2]):]
            continue
        #如果是Move from
        if "M" in Input[0]:
            #[284, 292] To [282, 290] 把[284, 292]这8行移动到[282,290]
            croppedTxt = Input[0][10:]
            #from和to的4个数字分开.
            splitTxt = croppedTxt.split(' To ')
            valueArr = [item.split(', ') for item in splitTxt]
            moveData = (int(valueArr[0][0][1:]), 
                        int(valueArr[0][1][:-1]), 
                        int(valueArr[1][0][1:]), 
                        int(valueArr[1][1][:-1]))
            #位置不相等,如果位置相等,等于什么都没有做
            if moveData[0] != moveData[2]:
                #行号小于 
                if moveData[0] < moveData[2]:
                    essayText = essayText[:moveData[0]] + essayText[moveData[1]:moveData[3]] +\
                    essayText[moveData[0]:moveData[1]] + essayText[moveData[3]:]
                #行号大于
                else:
                    essayText = essayText[:moveData[2]] + essayText[moveData[0]:moveData[1]] +\
                    essayText[moveData[2]:moveData[0]] + essayText[moveData[1]:]
            continue
        #由于continue会跳过这个循环,这里应该是在Input[1]的位置'Input'了Input[2]
        essayText = essayText[:Input[1] - len(Input[2])] + Input[2] + essayText[Input[1] - len(Input[2]):]     
            
    return USER_ID, essayText
AGGREGATIONS = ['count','min','max','first','last', 'median','sum','std']

#将论文转成单词
def word_feats(df):
    essay_df = df
    #对空格,\n,句号问号感叹号,逗号进行匹配,得到一个拆分后的列表.
    essay_df['word'] = essay_df['essay'].apply(lambda x: re.split(' |\\n|\\.|\\?|\\!|\\,',x))
    # essay1 [1,2,3] essay2[4,5] ->5行 essay1 1  // essay1 2 // essay1 3 // essay2 1 // essay2 2
    essay_df = essay_df.explode('word')
    #求出每个单词的长度
    essay_df['word_len'] = essay_df['word'].apply(lambda x: len(x))
    #去掉单词长度为0的数据,应该是标点符号和换行符之间,换行符与空格之间之类的.
    word_df = essay_df[essay_df['word_len'] != 0]
   
    #根据id计算单词长度的统计学变量
    word_agg_df = word_df[['id','word_len']].groupby(['id']).agg(AGGREGATIONS)
    #比如('mean','word_len')->'mean_word_len'
    word_agg_df.columns = ['_'.join(x) for x in word_agg_df.columns]
    word_agg_df['id'] = word_agg_df.index
#     for word_l in [5, 6, 7, 8, 9, 10, 11, 12]:
#         #ge 就是Latex里>=的符号,筛选出word_len>=word_l的行,根据id进行统计,提取每个计数的第0行
#         word_agg_df[f'word_len_ge_{word_l}_count'] = word_df[word_df['word_len'] >= word_l].groupby(['id']).count().iloc[:, 0]
#         #如果有缺失值就填充为0
#         word_agg_df[f'word_len_ge_{word_l}_count'] = word_agg_df[f'word_len_ge_{word_l}_count'].fillna(0)
    #重置索引
    word_agg_df = word_agg_df.reset_index(drop=True)
    return word_agg_df

#句子特征
def sent_feats(df):
    essay_df = df#传入的df就是论文的df
    #对句子按照. ? !进行拆分. 得到一个拆分后的列表.
    essay_df['sent'] = essay_df['essay'].apply(lambda x: re.split('\\.|\\?|\\!',x))
    # essay1 [1,2,3] essay2[4,5] ->5行 essay1 1  // essay1 2 // essay1 3 // essay2 1 // essay2 2
    essay_df = essay_df.explode('sent')
    #将换行符'\n'变成空白字符 strip 去除行头和行尾的空白字符.
    essay_df['sent'] = essay_df['sent'].apply(lambda x: x.replace('\n','').strip())
    #统计一下每个句子的长度 
    essay_df['sent_len'] = essay_df['sent'].apply(lambda x: len(x))
    #求一下每个句子单词的个数.
    essay_df['sent_word_count'] = essay_df['sent'].apply(lambda x: len(x.split(' ')))
    #去掉那些句子长度为0的数据
    df = essay_df[essay_df.sent_len!=0].reset_index(drop=True)
    
    #统计句子长度的统计学变量和每个句子词数的统计学变量
    sent_agg_df = pd.concat(
        [df[['id','sent_len']].groupby(['id']).agg(AGGREGATIONS), df[['id','sent_word_count']].groupby(['id']).agg(AGGREGATIONS)], axis=1
    )
    #比如('mean','sent_len')->'mean_sent_len'
    sent_agg_df.columns = ['_'.join(x) for x in sent_agg_df.columns]
    sent_agg_df['id'] = sent_agg_df.index
    
#     # New features intoduced here: https://www.kaggle.com/code/mcpenguin/writing-processes-to-quality-baseline-v2
#     for sent_l in [50, 60, 75, 100]:
#         #ge 就是Latex里>=的符号,筛选出sent_len>=sent_l的行,根据id进行统计,提取每个计数的第0行
#         sent_agg_df[f'sent_len_ge_{sent_l}_count'] = df[df['sent_len'] >= sent_l].groupby(['id']).count().iloc[:, 0]
#         #如果有缺失值就填充为0
#         sent_agg_df[f'sent_len_ge_{sent_l}_count'] = sent_agg_df[f'sent_len_ge_{sent_l}_count'].fillna(0)
    #重置索引
    sent_agg_df = sent_agg_df.reset_index(drop=True)
    #一句话里词的个数的count,其实就是有多少句话,也就是sent_len的count.重复了,故去掉.
    sent_agg_df.drop(columns=["sent_word_count_count"], inplace=True)
    #sent_len_count其实就是有多少句话,故rename.
    sent_agg_df = sent_agg_df.rename(columns={"sent_len_count":"sent_count"})
    return sent_agg_df

def parag_feats(df):
    essay_df = df
    #按照'\n'划分成段落 [1,2,3]
    essay_df['paragraph'] = essay_df['essay'].apply(lambda x: x.split('\n'))
    #[论文1 [段落1 段落2,……]->[论文1 段落1 // 论文1 段落2]
    essay_df = essay_df.explode('paragraph')
    #统计段落的长度
    essay_df['paragraph_len'] = essay_df['paragraph'].apply(lambda x: len(x)) 
    #统计每个段落的词数
    essay_df['paragraph_word_count'] = essay_df['paragraph'].apply(lambda x: len(x.split(' ')))
    essay_df['paragraph_sent_count'] = essay_df['paragraph'].apply(lambda x: len(re.split('\\.|\\?|\\!',x)))
    #将段落长度>2的文本留下来.
    df = essay_df[essay_df.paragraph_len>2].reset_index(drop=True)

    paragraph_agg_df = pd.concat(
        [df[['id','paragraph_len']].groupby(['id']).agg(AGGREGATIONS), 
         df[['id','paragraph_word_count']].groupby(['id']).agg(AGGREGATIONS),
         df[['id','paragraph_sent_count']].groupby(['id']).agg(AGGREGATIONS)
        ], axis=1
    ) 
    paragraph_agg_df.columns = ['_'.join(x) for x in paragraph_agg_df.columns]
    paragraph_agg_df['id'] = paragraph_agg_df.index
    paragraph_agg_df = paragraph_agg_df.reset_index(drop=True)
    #段落词数的数量就是段落的数量,也就是paragraph_len_count,段落句子数的数量同理,故drop
    paragraph_agg_df.drop(columns=["paragraph_word_count_count", "paragraph_sent_count_count"], inplace=True)
    paragraph_agg_df = paragraph_agg_df.rename(columns={"paragraph_len_count":"paragraph_count"})
    return paragraph_agg_df

#文本的自动可读性指数 旨在衡量文本的可理解性.输出是理解课文所需的美国年级水平的近似表示.
#https://www.nhooo.com/note/qa0tpe.html
#初步理解:相同词数的情况下,句子越少,说明句子相对来说会很长,越长越不容易理解.words/sentence就会越大.
#字符数相同的情况下,词数越多,单词越短,短的单词可能简单,所以就好理解.characters/words变小.
#数值小就好理解,数值大就不好理解.具体的公式可能用数据做过实验得出?
def ARI(txt):
    characters=len(txt)
    words=len(re.split(' |\\n|\\.|\\?|\\!|\,',txt))#空格,换行符,句号,问号,感叹号,逗号分开.
    sentence=len(re.split('\\.|\\?|\\!',txt))#句号,问号,感叹号分开的句子.
    ari_score=4.71*(characters/words)+0.5*(words/sentence)-21.43
    return ari_score
"""
http://www.supermagnus.com/mac/Word_Counter/index.html
McAlpine EFLAW© Test
     (W + SW) / S
McAlpine EFLAW© Readability
     Scale:
     1-20: Easy
     21-25: Quite Easy
     26-29: Mildly Difficult
     ≥ 30: Very Confusing
     S:total sentences
     W:total words
"""
def McAlpine_EFLAW(txt):
    W=len(re.split(' |\\n|\\.|\\?|\\!|\,',txt))#空格,换行符,句号,问号,感叹号,逗号分开.
    S=len(re.split('\\.|\\?|\\!',txt))#句号,问号,感叹号分开的句子.
    mcalpine_eflaw_score=(W+S*W)/S
    return mcalpine_eflaw_score
"""
https://readable.com/readability/coleman-liau-readability-index/

=0.0588*L-0.296*S-15.8
L是每100个单词有多少个字母,S是平均每100个单词有多少句子.
"""
def CLRI(txt):
    characters=len(txt)
    words=len(re.split(' |\\n|\\.|\\?|\\!|\,',txt))#空格,换行符,句号,问号,感叹号,逗号分开.
    sentence=len(re.split('\\.|\\?|\\!',txt))#句号,问号,感叹号分开的句子.
    L=100*characters/words
    S=100*sentence/words
    clri_score=0.0588*L-0.296*S-15.8
    return clri_score
#获取文本块的特征.
def get_text_chunk_features(df):
    punctuation = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
    #计算论文的长度
    df['text_length'] = df['essay'].apply(len)

    #计算换行符的个数.
    df['num_newlines'] = df['essay'].apply(lambda x: x.count('\n'))
    #文本可读性的3个指标
    df['automated_readability_index'] = df['essay'].apply(ARI)
    df['mcalpine_eflaw'] = df['essay'].apply(McAlpine_EFLAW)        
    df['coleman_liau'] = df['essay'].apply(CLRI)

    #字符‘q’在总字符数中的占比,其他的可能是空格和标点符号
    df['repetitiveness'] = df['essay'].apply(lambda x: x.count('q') / max(len(x), 1))

    #平均词长度=所有的词的总长度/总词数
    df['avg_word_length'] = df['essay'].apply(lambda x: sum(len(word) for word in x.split()) / max(1, len(x.split())))
    #单词紧急多样性 set(x.split())=['qq','qqq','qqqq'……],len(x.split())是论文总词数
    df['word_lexical_diversity'] = df['essay'].apply(lambda x: len(set(x.split())) / len(x.split()))
        
    #单引号和双引号的使用次数.
    df['num_s_quotations'] = df['essay'].apply(lambda x: x.count("'"))
    df['num_d_quotations'] = df['essay'].apply(lambda x: x.count('"'))
    #问号、感叹号,逗号,句号的使用次数
    df['qm_count'] = df['essay'].apply(lambda x: x.count('?'))
    df['excm_count'] = df['essay'].apply(lambda x: x.count('!'))
    df['comma_count'] = df['essay'].apply(lambda x: x.count(','))
    df['dot_count'] = df['essay'].apply(lambda x: x.count('.'))
    #冒号和分号的使用次数
    df['num_prelist_count'] = df['essay'].apply(lambda x: x.count(':')) +\
                            df['essay'].apply(lambda x: x.count(";"))    
    
    #空白字符例如:空格,换行符,制表符后面有一个句号的错误.
    df["space_n_dot_mistake"] = df['essay'].apply(lambda x: len(re.findall(r'\s\.', x)))
    #空白字符例如:空格,换行符,制表符后面有一个逗号的错误.
    df["space_n_comma_mistake"] = df['essay'].apply(lambda x: len(re.findall(r'\s\,', x)))
    #空白字符例如:空格,换行符,制表符前面有一个句号的错误.
    df["comma_n_nonspace_mistake"] = df['essay'].apply(lambda x: len(re.findall(r'\,\S', x)))
    #空白字符例如:空格,换行符,制表符前面有一个逗号的错误.
    df["dot_n_nonspace_mistake"] = df['essay'].apply(lambda x: len(re.findall(r'\.\S', x)))
    #总共有几个错误,对错误做一个汇总.
    df["total_punc_mistake"] = (
        df["space_n_dot_mistake"] +
        df["space_n_comma_mistake"] +
        df["comma_n_nonspace_mistake"] +
        df["dot_n_nonspace_mistake"]
    )
    #标点符号犯错误的次数/4种标点符号的使用次数(问号和感叹号出现次数不多)
    df["punc_mistake_ratio"] = df["total_punc_mistake"] / (df['qm_count'] +
                                                           df['excm_count'] +
                                                           df['comma_count'] +
                                                           df['dot_count'])

    #统计不同的单词种类. 
    df['unique_word_count'] = df['essay'].apply(lambda x: len(set(re.findall(r'\w+', x.lower()))))
    
    #统计每种标点符号出现的次数和
    df['punctuation_count'] = df['essay'].apply(lambda x: sum(x.count(p) for p in punctuation)) 

    return df

#对一段文本txt进行处理的函数
def standardize_text(txt):
    txt = re.sub(r'\t' , '', txt)#将文本中的制表符替换成空字符串
    txt = re.sub(r'\n {1,}' , '\n', txt)#换行符后面如果有空格,去掉
    txt = re.sub(r' {1,}\n' , '\n', txt)#换行符前面有空格的话去掉
    txt = re.sub(r'\n{2,}' , '\n', txt)#如果有2个以上的换行符,替换成单个换行符
    txt = re.sub(r' {2,}' , ' ', txt)#如果有出现连续2个空格,替换成单个空格.
    txt = txt.strip()#开头和结尾如果有空白字符去掉
    return txt#返回经过处理的文本
def TextProcessor(inp_df):

    for rowi in range(len(inp_df)):
        #如果一篇论文删除空格后什么都没有,替换为'q',好心假装你写了东西.
        if inp_df.loc[rowi, "essay"].replace(" ", "") == "":
            inp_df.loc[rowi, "essay"] = "q"   
    
    #对传入的论文去掉一些多余的字符.
    inp_df["essay"] = inp_df["essay"].apply(lambda x: standardize_text(txt=x))
    
    #获取文本块的相关特征.
    print("creating complete features")
    inp_df = get_text_chunk_features(inp_df) 
    #获取文本的单词,句子,段落特征
    wf_df = word_feats(inp_df)
    sf_df = sent_feats(inp_df)
    pf_df = parag_feats(inp_df)
    #将论文
    inp_df = inp_df.merge(wf_df, how="left", on="id")
    inp_df = inp_df.merge(sf_df, how="left", on="id")
    inp_df = inp_df.merge(pf_df, how="left", on="id")
    #提取好特征,把论文,词语,句子,段落去掉
    inp_df.drop(["essay", "word", "sent", "paragraph"],axis=1,inplace=True)
    
    return inp_df

#数值列,activity,event和text_change的重要类别
num_cols = ['down_time', 'up_time', 'action_time', 'cursor_position', 'word_count', 'event_id']
activities = ['Input', 'Remove/Cut', 'Nonproduction', 'Replace', 'Paste']
events = ['q', 'Space', 'Backspace', 'Shift', 'ArrowRight', 'Leftclick', 'ArrowLeft', '.', ',', 'ArrowDown', 'ArrowUp', 'Enter', 'CapsLock', "'", 'Delete', 'Unidentified']
text_changes = ['q', ' ', '.', ',', '\n', "'", '"', '-', '?', ';', '=', '/', '\\', ':']

#df表格的colname列统计values的count.
def count_by_values(df, colname, values):
    #maintain_order=True保持原有顺序
    fts = df.select(pl.col('id').unique(maintain_order=True))
    for i, value in enumerate(values):
        #根据每个id判断colname是不是value并统计个数,rename成colname_i_cnt
        tmp_df = df.group_by('id').agg(pl.col(colname).is_in([value]).sum().alias(f'{colname}_{i}_cnt'))
        #加上这个特征
        fts  = fts.join(tmp_df, on='id', how='left') 
    return fts
#暂停状态的特征聚合
def pause_stat_aggregator(df, prefix="iw"):
    temp = df.group_by("id").agg(
           #根据id计算time_diff的最大,中位数,均值,最小值,方差,sum
           pl.max('time_diff').alias(f"{prefix}_max_pause_time"),
           pl.median('time_diff').alias(f"{prefix}_median_pause_time"),
           pl.mean('time_diff').alias(f"{prefix}_mean_pause_time"),
           pl.min('time_diff').alias(f"{prefix}_min_pause_time"),
           pl.std('time_diff').alias(f"{prefix}_std_pause_time"),
           pl.sum('time_diff').alias(f"{prefix}_total_pause_time"),
           # time_diff在(0.5,1) (1,2) (2,3) >3的count.
           pl.col('time_diff').filter((pl.col('time_diff') > 0.5) & (pl.col('time_diff') <= 1)).count().alias(f"{prefix}_pauses_half_sec"),
           pl.col('time_diff').filter((pl.col('time_diff') > 1) & (pl.col('time_diff') <= 2)).count().alias(f"{prefix}_pauses_1_sec"),
           pl.col('time_diff').filter((pl.col('time_diff') > 2) & (pl.col('time_diff') <= 3)).count().alias(f"{prefix}_pauses_2_sec"),
           pl.col('time_diff').filter(pl.col('time_diff') > 3).count().alias(f"{prefix}_pauses_3_sec")
    )
    return temp  

def dev_feats(df):
    
    print("< Count by values features >")
    
    #统计activity,text_change,down_event,up_event这几个类别型变量的count down_event和up_event基本一样.
    feats = count_by_values(df, 'activity', activities)
    feats = feats.join(count_by_values(df, 'text_change', text_changes), on='id', how='left') 
    feats = feats.join(count_by_values(df, 'down_event', events), on='id', how='left') 

    print("< Numerical columns features >")

    #对action_time求和,对数值型变量求均值,方差,中位数,最小值,最大值,50%的数字 没有pl.mean(num_cols).suffix('_mean')和pl.quantile(num_cols, 0.5).suffix('_quantile')
    temp = df.group_by("id").agg(pl.sum('action_time').suffix('_sum'),pl.std(num_cols).suffix('_std'),
                                 pl.median(num_cols).suffix('_median'), pl.min(num_cols).suffix('_min'),
                                 pl.max(num_cols).suffix('_max'),
                                 )
    feats = feats.join(temp, on='id', how='left') 
    
    print("< Categorical columns features >")
    #类别型变量求了n_unique,加入特征.
    temp  = df.group_by("id").agg(pl.n_unique(['activity', 'down_event', 'up_event', 'text_change']))
    feats = feats.join(temp, on='id', how='left') 
    
    
    print("< Creating pause features >")   
    #up_time向后移动命名为up_time_lagged
    temp = df.with_columns(pl.col('up_time').shift().over('id').alias('up_time_lagged'))
    #按秒来计算停歇的时间 time_diff
    temp = temp.with_columns((abs(pl.col('down_time') - pl.col('up_time_lagged')) / 1000).fill_null(0).alias('time_diff'))
    #按空格键,句号,Enter(换行) ,分别是单词,句子,段落之间的标志.
    temp = temp.with_columns((pl.col("up_event") == "Space").alias("is_space"))
    temp = temp.with_columns((pl.col("up_event") == ".").alias("is_dot"))
    temp = temp.with_columns((pl.col("up_event") == "Enter").alias("is_enter"))

    temp = temp.with_columns(
        #根据id计算单词之间的累加和,如果缺失向后填充
        pl.col("is_space").cumsum().shift().backward_fill().over("id").alias("word_id"),
        #根据id计算句子之间的累加和,如果缺失向后填充        
        pl.col("is_dot").cumsum().shift().backward_fill().over("id").alias("sentence_id"),   
        #根据id计算段落之间的累加和,如果缺失向后填充   
        pl.col("is_enter").cumsum().shift().backward_fill().over("id").alias("paragraph_id"),    
    )
    
    #选择 activity为Input和Remove/cut的数据
    temp = temp.filter(pl.col('activity').is_in(['Input', 'Remove/Cut']))

    #All in 全部数据计算time_diff的统计特征
    iw_df = pause_stat_aggregator(df=temp, prefix="iw")

    # Between-words pauses   每个'id'的每个‘word_id’的第一个time_diff  计算统计特征
    bww_df = temp.group_by("id", "word_id").agg(pl.col("time_diff").first())
    bww_df = pause_stat_aggregator(df=bww_df, prefix="bww")

    # Between-sentences pauses 每个'id'的每个‘sentence_id’的第一个time_diff  计算统计特征
    bws_df = temp.group_by("id", "sentence_id").agg(pl.col("time_diff").first())
    bws_df = pause_stat_aggregator(df=bws_df, prefix="bws")

    # Between-paragraphs pauses 每个'id'的每个‘paragraph_id’的第一个time_diff  计算统计特征
    bwp_df = temp.group_by("id", "paragraph_id").agg(pl.col("time_diff").first())
    bwp_df = pause_stat_aggregator(df=bwp_df, prefix="bwp")

    #将所有特征拼接在一起
    feats = (feats.join(iw_df, on="id", how="left")
                 .join(bww_df, on="id", how="left")
                 .join(bws_df, on="id", how="left")
                 .join(bwp_df, on="id", how="left")
            )
    
    feats=feats.to_pandas()
    
    return feats

#统计每秒有几个['Input', 'Remove/Cut']的行为.
def get_keys_pressed_per_second(logs):
    #logs中为['Input', 'Remove/Cut']的event_id的个数
    temp_df = logs[logs['activity'].isin(['Input', 'Remove/Cut'])].groupby(['id']).agg(keys_pressed=('event_id', 'count')).reset_index()
    #每个id最小的down_time和最大的up_time
    temp_df_2 = logs.groupby(['id']).agg(min_down_time=('down_time', 'min'), max_up_time=('up_time', 'max')).reset_index()
    #按照id融合在一起
    temp_df = temp_df.merge(temp_df_2, on='id', how='left')
    #每秒有几个event_id
    temp_df['keys_per_second'] = temp_df['keys_pressed'] / ((temp_df['max_up_time'] - temp_df['min_down_time']) / 1000)
    return temp_df[['id', 'keys_per_second']]

    

def burst_features(df, burst_type="p"):
    #up_time向后移动命名为up_time_lagged
    temp = df.with_columns(pl.col('up_time').shift().over('id').alias('up_time_lagged'))
    #按秒来计算停歇的时间 time_diff
    temp = temp.with_columns((abs(pl.col('down_time') - pl.col('up_time_lagged')) / 1000).fill_null(0).alias('time_diff'))
    #如果P-burst,统计的是activity=Input,如果是R-burst,统计的是Remove/cut
    if burst_type == "p":
        temp = temp.with_columns(pl.col('activity').is_in(['Input']))
    elif burst_type == "r":
        temp = temp.with_columns(pl.col('activity').is_in(['Remove/Cut']))
    #action_time按秒来计算
    temp = temp.with_columns((pl.col('action_time') / 1000).alias("action_time_s"))
    #up_time按照秒来计算
    temp = temp.with_columns((pl.col('up_time') / 1000).alias("up_time_s"))
    #增加f'{burst_type}_burst_group'列,activity!='Input'是nan.
    #activity=='Input',有具体数值,上一个是Input的话,数值一样,上一个是nan的话,在上一次出现‘Input’的数值上+2.
    temp = temp.with_columns(pl.when(pl.col("activity")).then(pl.col("activity").rle_id()).alias(f'{burst_type}_burst_group'))
    #去掉nan的行.就是去掉activity不是'Input'.
    temp = temp.drop_nulls()

    #每个id,每个f"{burst_type}_burst_group" ==1,==3,==5…… 
    temp = temp.group_by("id", f"{burst_type}_burst_group").agg(
                                    #出现了几个activity
                                    pl.count('activity').alias(f'{burst_type}_burst_group_keypress_count'),
                                    #action_time的求和,均值,方差
                                    pl.sum('action_time_s').alias(f'{burst_type}_burst_group_timespent'),
                                    pl.mean('action_time_s').alias(f'{burst_type}_burst_keypress_timespent_mean'),
                                    pl.std('action_time_s').alias(f'{burst_type}_burst_keypress_timespent_std'),
                                    #这个burst_goup的up_time的最小值和最大值,即:第一次和最后一次.
                                    pl.min('up_time_s').alias(f'{burst_type}_burst_keypress_timestamp_first'),
                                    pl.max('up_time_s').alias(f'{burst_type}_burst_keypress_timestamp_last')
    )

    #每个id都有很多f"{burst_type}_burst_group",故还需要求统计特征.
    temp = temp.group_by("id").agg(
                #每个id出现连续的burst_group的求和,均值,方差,最大值.
                pl.sum(f'{burst_type}_burst_group_keypress_count').alias(f'{burst_type}_burst_keypress_count_sum'),
                pl.mean(f'{burst_type}_burst_group_keypress_count').alias(f'{burst_type}_burst_keypress_count_mean'),
                pl.std(f'{burst_type}_burst_group_keypress_count').alias(f'{burst_type}_burst_keypress_count_std'),
                pl.max(f'{burst_type}_burst_group_keypress_count').alias(f'{burst_type}_burst_keypress_count_max'),
                #每个id出现连续的burst_group的action_time的总时间求和,均值,方差,最大值
                pl.sum(f'{burst_type}_burst_group_timespent').alias(f'{burst_type}_burst_timespent_sum'),
                pl.mean(f'{burst_type}_burst_group_timespent').alias(f'{burst_type}_burst_timespent_mean'),
                pl.std(f'{burst_type}_burst_group_timespent').alias(f'{burst_type}_burst_timespent_std'),
                pl.max(f'{burst_type}_burst_group_timespent').alias(f'{burst_type}_burst_timespent_max'),
                #每个id出现连续的burst_group的action_time的平均时间求均值
                pl.mean(f'{burst_type}_burst_keypress_timespent_mean').alias(f'{burst_type}_burst_keypress_timespent_mean'),
                #每个id出现连续的burst_group的action_time的方差求平均
                pl.mean(f'{burst_type}_burst_keypress_timespent_std').alias(f'{burst_type}_burst_keypress_timespent_std'),
                #每个id的burst开始的时间,每个id的burst结束的时间.
                pl.min(f'{burst_type}_burst_keypress_timestamp_first').alias(f'{burst_type}_burst_keypress_timestamp_first'),
                pl.max(f'{burst_type}_burst_keypress_timestamp_last').alias(f'{burst_type}_burst_keypress_timestamp_last')
    )
    
    #将polars对象转成pandas对象.
    temp = temp.to_pandas()
    return temp


def Preprocessor(logs):
    #将数据从pandas对象转成polars对象
    pl_logs = pl.from_pandas(logs)

    print("< Creating keys_pressed_per_second features >")                
    feat_df = get_keys_pressed_per_second(logs)#统计每秒有几个Input或者Remove/cut操作
             
    feat_df = feat_df.merge(dev_feats(df=pl_logs), how="left", on="id") 
    
    print("< Creating PR-Burst features >")  
    #这里是P-burst和R-burst
    feat_df = feat_df.merge(burst_features(df=pl_logs, burst_type="p"), how="left", on="id")    
    feat_df = feat_df.merge(burst_features(df=pl_logs, burst_type="r"), how="left", on="id")    
    
    #获取每个id的论文
    essays = logs.groupby("id").apply(get_Essays)
    essays = pd.DataFrame(essays.tolist(), columns=["id", "essay"])
    essay_feats = TextProcessor(essays)
    feat_df=feat_df.merge(essay_feats,how="left", on="id")
    
    #每秒花费多少时间在P-burst和R-burst上
    feat_df["p_bursts_time_ratio"] = feat_df["p_burst_timespent_sum"] / (feat_df["up_time_max"] / 1000)
    feat_df["r_bursts_time_ratio"] = feat_df["r_burst_timespent_sum"] / (feat_df["up_time_max"] / 1000)
    #action_time在写作总时间的占比情况
    feat_df["action_time_ratio"] = feat_df["action_time_sum"] / feat_df["up_time_max"]
    #所有数据的停顿时间在总时间的占比
    feat_df["pause_time_ratio"] = feat_df["iw_total_pause_time"] / (feat_df["up_time_max"] / 1000)
    #平均每秒发生几次停顿2~3秒
    feat_df["pausecount_time_ratio"] = feat_df["iw_pauses_2_sec"] / (feat_df["up_time_max"] / 1000)
    #平均每秒写多少词数
    feat_df['word_time_ratio'] = feat_df['word_count_max'] / (feat_df["up_time_max"] / 1000)
    #平均每个event写多少词数
    feat_df['word_event_ratio'] = feat_df['word_count_max'] / feat_df["event_id_max"] 
    #平均每秒多少个event
    feat_df['event_time_ratio'] = feat_df['event_id_max']  / (feat_df["up_time_max"] / 1000)
    #文本长度/秒数,即:每秒写多少文本.
    feat_df["text_length_time_ratio"] = feat_df["text_length"] / (feat_df["up_time_max"] / 1000)
    
    return feat_df

train_logs=pd.read_csv("/kaggle/input/linking-writing-processes-to-writing-quality/train_logs.csv")
print(f"len(train_logs):{len(train_logs)}")
train_logs=train_logs.sort_values(by=['id', 'down_time'])
# 重置索引
train_logs = train_logs.reset_index(drop=True)
# 根据'id'列进行分组，并为每个分组添加一个递增的序列
train_logs['event_id'] = train_logs.groupby('id').cumcount() + 1

train_scores=pd.read_csv("/kaggle/input/linking-writing-processes-to-writing-quality/train_scores.csv")

test_logs=pd.read_csv("/kaggle/input/linking-writing-processes-to-writing-quality/test_logs.csv")
print(f"len(test_logs):{len(test_logs)}")
test_logs=test_logs.sort_values(by=['id', 'down_time'])
# 重置索引
test_logs = test_logs.reset_index(drop=True)
# 根据'id'列进行分组，并为每个分组添加一个递增的序列
test_logs['event_id'] = test_logs.groupby('id').cumcount() + 1

print("feature engineer")
train_feats = Preprocessor(train_logs)
train_feats = train_feats.merge(train_scores, how="left", on="id")
test_feats = Preprocessor(test_logs)

#找到只有唯一值的列,删掉
keys=train_feats.keys().values
unique_cols=[key for key in keys if train_feats[key].nunique()<2]
print(f"drop unique_cols:{unique_cols}")
train_feats = train_feats.drop(columns=unique_cols)
test_feats = test_feats.drop(columns=unique_cols)
#将里面正无穷和负无穷的值替换成缺失值
train_feats.replace([np.inf, -np.inf], np.nan, inplace=True)
test_feats.replace([np.inf, -np.inf], np.nan, inplace=True)
train_feats.drop(['id'],axis=1,inplace=True)
print(f"total_feats_counts:{len(test_feats.keys().values)}")


#这里创建了lgbm,cat,xgb模型
def make_model():
    
    #大佬找好的参数,这里不做改动
    cat_params = {'learning_rate': 0.024906985231770738, 'depth': 5, 
                  'l2_leaf_reg': 3.7139894959529283, 'subsample': 0.18527466886647015, 
                  'colsample_bylevel': 0.6552973951000719, 'min_data_in_leaf': 93,
                "silent": True,"iterations": 1000, "random_state": seed,"use_best_model":False
                 }
    
    lgb_params={'reg_alpha': 1.0894488472899402, 'reg_lambda': 6.290929934336985,
                'colsample_bytree': 0.6218522907548012, 'subsample': 0.9579924238280629, 
                'learning_rate': 0.0027076430412427566, 'max_depth': 8, 'num_leaves': 947, 
                 'min_child_samples': 57,'n_estimators': 2500,'metric': 'rmse',
                'random_state': seed,'verbosity': -1,'force_col_wise': True
                }
    
    xgb_params={'max_depth': 2, 'learning_rate': 0.009998236038809146,
                'n_estimators': 1000, 'min_child_weight': 17,
                'gamma': 0.1288249858838246, 'subsample': 0.5078057280148618,
                'colsample_bytree': 0.7355762136239921, 'reg_alpha': 0.670956206987811,
                'reg_lambda': 0.06818351284100388, 'random_state': seed
               }
    
    model1 = LGBMRegressor(**lgb_params)
    
    model2 = CatBoostRegressor(**cat_params)
    
    model3 = XGBRegressor(**xgb_params)
    
    models = []
    models.append((model1, 'lgb'))
    models.append((model2, 'cat'))
    models.append((model3, 'xgb'))
    
    return models
#评估指标是RMSE
def RMSE(y_true,y_pred):
    return np.sqrt(np.mean((y_true-y_pred)**2))

X=train_feats.drop(['score'],axis=1)
y=train_feats['score']

models_and_errors_dict = {}

y_hats = dict()

#设置submission_df,id和score
submission_df = pd.DataFrame(test_feats['id'])
submission_df['score'] = 3.5#如果报错,将预测结果设置为3.5

#取出test_feats中所有列
X_unseen = test_feats.drop(['id'],axis=1).copy()

num_folds=10

for model, model_type in make_model():
    
    oof_pred=np.zeros((len(y)))
    
    y_hats[model_type] = []#某个model的预测结果
        
    #10折交叉验证
    skf = StratifiedKFold(n_splits=num_folds,random_state=seed, shuffle=True)

    for fold, (train_index, valid_index) in (enumerate(skf.split(X, y.astype(str)))):

        # Split data into train and test sets
        X_train, X_test = X.iloc[train_index], X.iloc[valid_index]
        y_train, y_test = y.iloc[train_index], y.iloc[valid_index]

        X_train_copy, X_test_copy = X_train.copy(), X_test.copy()

        model.fit(X_train_copy, y_train)

        #做预测
        y_hat = model.predict(X_test_copy)
        
        oof_pred[valid_index]=y_hat
        #计算RMSE
        rmse = RMSE(y_test, y_hat)
        print(f'RMSE: {rmse} on fold {fold}')
        
        
        #复制是因为有的要归一化
        X_unseen_copy = X_unseen.copy()
        X_unseen_copy=X_unseen_copy
        
        y_hats[model_type].append(model.predict(X_unseen_copy))
        #在字典里没有就创建一个
        if model_type not in models_and_errors_dict:
            models_and_errors_dict[model_type] = []
        models_and_errors_dict[model_type].append((model, rmse, None, None,oof_pred))
for key in y_hats.keys():
    #如果有值的话,求平均,赋值给submission_df
    if y_hats[key]:
        y_hat_avg = np.mean(y_hats[key], axis=0)
        submission_df['score_' + key] = y_hat_avg
submission_df.head()


#融合模型的权重
blending_weights = {
    'lgb': 0.4,
    'cat': 0.4,
    'xgb': 0.2,
}
lgb_oof_pred=models_and_errors_dict['lgb'][num_folds-1][4]
cat_oof_pred=models_and_errors_dict['cat'][num_folds-1][4]
xgb_oof_pred=models_and_errors_dict['xgb'][num_folds-1][4]
margin=1000
target=y.values
current_RMSE=RMSE(target,(lgb_oof_pred+cat_oof_pred+xgb_oof_pred)/3)
best_i=0
best_j=0
for i in range(0,margin):
    for j in range(0,margin-i):
        #k=1000-i-j
        blend_oof_pred=(i*lgb_oof_pred+j*cat_oof_pred+(margin-i-j)*xgb_oof_pred)/margin
        if RMSE(target,blend_oof_pred)<current_RMSE:
            current_RMSE=RMSE(target,blend_oof_pred)
            best_i=i
            best_j=j
#找到最好的参数之后
blending_weights['lgb']=best_i/margin
blending_weights['cat']=best_j/margin
blending_weights['xgb']=(margin-best_i-best_j)/margin
print(f"current_RMSE:{current_RMSE},blending_weights:{blending_weights}")
print("blending")
blended_score=np.zeros((len(test_feats)))
for k, v in blending_weights.items():
    blended_score += submission_df['score_' + k] * v
print(f"blended_score:{blended_score}")


submission=pd.read_csv("/kaggle/input/linking-writing-processes-to-writing-quality/sample_submission.csv")
submission['score']=blended_score
submission.to_csv("submission.csv",index=None)
submission.head()