concatenated_data_processing.py

# -*- coding: utf-8 -*-
"""concatenated_data_processing

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1HBNUWe6IwI9YCExCqgIECcjTJj0GWYKd
"""

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import seaborn
import plotly.express as px
import plotly
from scipy.stats import ttest_1samp
import plotly.graph_objects as go

from pandas import json_normalize
from scipy import stats
from sklearn import preprocessing
from factor_analyzer import FactorAnalyzer
from factor_analyzer.factor_analyzer import calculate_kmo, calculate_bartlett_sphericity
from PIL import Image
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.metrics import mean_absolute_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import MinMaxScaler

plt.rcParams['font.size'] = 18  # Default font size
plt.rcParams['axes.labelsize'] = 20  # Font size of axis labels
plt.rcParams['axes.titlesize'] = 20  # Font size of plot titles
plt.rcParams['xtick.labelsize'] = 16  # Font size of x-axis tick labels
plt.rcParams['ytick.labelsize'] = 16  # Font size of y-axis tick labels
plt.rcParams['legend.fontsize'] = 18  # Font size of legend

excel_file_path = 'concatenated_data.xlsx'
scenario = "1_2_3"  # Choose between 1_2, 3, 1_2_3
group = ['G3', 'G4']  # 'G1', 'G2', 'G3', 'G4'

# Function to format participant_number as XXX
# def format_participant_number(participant):
#     if isinstance(participant, str) and participant.startswith('P') and participant[1:].isdigit():
#         numeric_part = participant[1:]
#         formatted_numeric_part = numeric_part.zfill(3)
#         return f'P{formatted_numeric_part}'  # Assumes participant_number is in the format 'PXX'
#     else:
#         numeric_part = participant[1:]
#         formatted_numeric_part = numeric_part.zfill(3)
#         return f'T{formatted_numeric_part}'  # Assumes participant_number is in the format 'PXX'
#
#
# # Load Excel file
# selected_sheets = ['simulator_logs',
#                    'indexes',
#                    'support_questions',
#                    'eye_tracking',
#                    'watch_data',
#                    'AI_vs_human',
#                    'AI_questions',
#                    ]
#
# # Read the first sheet to initialize the merged dataframe
# merged_df = pd.read_excel(excel_file_path, sheet_name=selected_sheets[0])
#
# # Iterate through the remaining sheets and merge based on common columns
# for sheet_name in selected_sheets[1:]:
#     df = pd.read_excel(excel_file_path, sheet_name=sheet_name)
#     merged_df = pd.merge(merged_df, df, on=['Participant', 'Group', 'Scenario'], how='outer')
#
# filtered_df = merged_df[merged_df['Group'].isin(group)]
# if scenario == "1_2":
#     filtered_df = filtered_df[filtered_df['Scenario'].isin(['S1', 'S2'])]
# if scenario == "3":
#     filtered_df = filtered_df[filtered_df['Scenario'].isin(['S3'])]
# filtered_df = filtered_df[~filtered_df['Participant'].str.startswith('T')]
#
# '''*********************'''
#
# filtered_df['Consequence'] = filtered_df['Consequence'].map(
#     {'Safe': 0,
#      'Impurity of air in tank atmosphere': 1,
#      'Plant Shutdown': 2,
#      'Reactor Overheated': 3,
#      })
# # filtered_df['Recovery_status'] = filtered_df['Recovery_status'].map({'Poor': 0, 'Good': 1, 'Optimal': 2})
# filtered_df['Overall_performance'] = filtered_df['Overall_performance'].map({'Poor': 0, 'Good': 1, 'Optimal': 2})
#
# filtered_df['Participant'] = filtered_df['Participant'].apply(format_participant_number)
# filtered_df.drop_duplicates(keep='first', inplace=True)
# filtered_df.sort_values(by=['Participant', 'Scenario'], ascending=[True, True], inplace=True, ignore_index=True)
#
# '''*********************'''
#
# columns_to_normalize = ['Pupil_diameter_overflow',
#                         'Pupil_diameter_critical',
#                         'Velocity_overflow',
#                         'Velocity_critical',
#                         'Fixation_duration_overflow',
#                         'Fixation_duration_critical',
#                         'Peak_velocity_overflow',
#                         'Peak_velocity_critical',
#                         'Saccade_duration_overflow',
#                         'Saccade_duration_critical',
#                         'Saccade_amplitude_overflow',
#                         'Saccade_amplitude_critical'
#                         ]
#
# for col in columns_to_normalize:
#     baseline_col = col.replace('_critical', '_baseline').replace('_overflow', '_baseline')
#     filtered_df[col] = filtered_df[col] / filtered_df[baseline_col]
#
# '''*********************'''
#
# numeric_columns = filtered_df.select_dtypes(include=['number']).columns
# filtered_norm_df = filtered_df.copy()
#
# grouped_by_scenario = filtered_norm_df.groupby('Scenario')
# scaler = MinMaxScaler()
# filtered_norm_df[numeric_columns] = grouped_by_scenario[numeric_columns].\
#     transform(lambda x: scaler.fit_transform(x.values.reshape(-1, 1)).flatten())
#
# filtered_norm_mean_df = filtered_norm_df.groupby(['Participant', 'Group'])[numeric_columns].mean().reset_index()
#
# filtered_norm_mean_df.to_excel(f'./merged_filtered_norm_mean_{group}_s{scenario}_data.xlsx')

filtered_norm_mean_df = pd.read_excel(f'merged_filtered_norm_mean_s1_2_3_data.xlsx')
columns_to_filter = [
    "Alarm_prioritization_support",
    "Alarm_lists_support",
    "Pupil_diameter_baseline",
    "Velocity_baseline",
    "Fixation_duration_baseline",
    "Saccade_duration_baseline",
    "Saccade_amplitude_baseline",
    "Peak_velocity_baseline",
    "Peak_velocity_overflow",
    "Peak_velocity_critical",
]
filtered_norm_mean_df = filtered_norm_mean_df.drop(columns=columns_to_filter)

numeric_columns = filtered_norm_mean_df.select_dtypes(include=['number']).columns
filtered_norm_mean_df['Group'] = filtered_norm_mean_df['Group'].map({'G3': 'GroupN', 'G4': 'GroupAI'})

'''Correlation'''
# combined_corr = filtered_norm_mean_df.corr(numeric_only=True)
# matrix = np.triu(combined_corr)
# plt.figure(figsize=(50, 50))
# sns.heatmap(combined_corr, annot=True, cmap='coolwarm', fmt=".2f", mask=matrix)
# plt.title('Correlation Matrix')
# plt.xticks(rotation=90)
# plt.subplots_adjust(bottom=0.1)
# plt.show()

combined_corr = filtered_norm_mean_df.corr(numeric_only=True)
thresh = 0.4
combined_corr = combined_corr[(combined_corr <= -thresh) | (combined_corr >= thresh)]

mask = np.tril(np.ones(combined_corr.shape), k=-1)
combined_corr = combined_corr.where(mask.astype(bool))

combined_corr.dropna(axis=0, how='all', inplace=True)
combined_corr.dropna(axis=1, how='all', inplace=True)

fig, ax = plt.subplots(figsize=(15, 15))
heatmap = sns.heatmap(combined_corr,
                      annot=True,
                      cmap='coolwarm',
                      fmt=".2f",
                      linewidths=0.1,
                      linecolor='grey',
                      annot_kws={'size': 8},
                      ax=ax
                      )

plt.title('Correlation Matrix')
plt.xticks(rotation=90)
plt.subplots_adjust(bottom=0.35, left=0.2)
plt.show()


'''P-value'''

columns = list(pd.read_excel(excel_file_path,
                             sheet_name=[
                                 'simulator_logs',
                                 'indexes',
                                 'support_questions',
                                 # 'watch_data',
                                 # 'eye_tracking'
                                 'AI_vs_human',
                                 'AI_questions',
                             ],
                             nrows=1).values())

flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
column_names = list(dict.fromkeys(flattened_list))
column_names = [column for column in column_names if column not in columns_to_filter]

filtered_norm_mean_df = filtered_norm_mean_df[column_names]

# Assuming filtered_norm_mean_df is your DataFrame
filtered_norm_mean_df = filtered_norm_mean_df.dropna()
# Select only numeric columns
numeric_columns_p = filtered_norm_mean_df.select_dtypes(include=['number'])

# Create an empty list to store results
results = []

# Iterate through numeric columns
for column in numeric_columns_p.columns:
    # Perform t-test against a hypothetical mean (e.g., 0)
    t_stat, p_value = ttest_1samp(numeric_columns_p[column], 0)

    # Append the results to the list
    results.append({'Variable': column, 'P-Value': p_value})

# Create a DataFrame from the list of results
p_values_df = pd.DataFrame(results)

# Sort the DataFrame by p-values for better visualization
p_values_df = p_values_df.sort_values(by='P-Value')

# Set the significance level (e.g., 0.05)
alpha = 0.05

# Create a bar plot with color-coded significance levels
plt.figure(figsize=(10, 6))
bar_plot = sns.barplot(x='P-Value', y='Variable', data=p_values_df, palette="coolwarm")

# Add significance level lines
plt.axvline(x=alpha, color='red', linestyle='--', label=f'Significance Level ({alpha})')

plt.subplots_adjust(left=0.2)

# Set labels and title
plt.xlabel('P-Value')
plt.ylabel('Variable')
plt.title('P-Values for Numeric Variables')
plt.legend()

# Show the plot
plt.show(block=True)

'''Radar Plot'''
columns = list(pd.read_excel(excel_file_path,
                             sheet_name=[
                                 'simulator_logs',
                                 'indexes',
                                 'support_questions',
                                 'watch_data',
                                 'eye_tracking',
                                 # 'AI_vs_human',
                                 # 'AI_questions',
                             ],
                             nrows=1).values())
flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
column_names = list(dict.fromkeys(flattened_list))
column_names = [column for column in column_names if column not in columns_to_filter]

filtered_norm_mean_wo_ai_df = filtered_norm_mean_df[column_names]

if scenario == "1_2" or "3":
    filtered_norm_mean_wo_ai_df.drop(columns=filtered_norm_mean_wo_ai_df.filter(regex='_overflow').columns,
                                     inplace=True)

numeric_columns_wo_ai = filtered_norm_mean_wo_ai_df.select_dtypes(include=['number']).columns

groupby_groups = pd.pivot_table(filtered_norm_mean_wo_ai_df.drop(columns=['Participant']), index=["Group"],
                                aggfunc="mean", sort=[False, False]).sort_values(by='Group', ascending=False)
df = pd.DataFrame(dict(
    value=groupby_groups.values.flatten(),
    variable=np.array(([numeric_columns_wo_ai.values] * 2)).flatten(),
    Group=np.array([['GroupN'] * len(numeric_columns_wo_ai), ['GroupAI'] * len(numeric_columns_wo_ai)]).flatten()
))
fig = px.line_polar(df,
                    r='value',
                    theta='variable',
                    line_close=True,
                    color='Group')
fig.update_traces(fill='toself')
fig.update_layout(legend=dict(
    yanchor="top",
    y=1.15,
    xanchor="center",
    x=0.5
))
fig.update_layout(font=dict(size=14))
plotly.offline.plot(fig)

'''Consequences'''
consequence_df = pd.read_excel('concatenated_data.xlsx', sheet_name='simulator_logs')
consequence_df = consequence_df[consequence_df['Group'].isin(['G3', 'G4'])]
consequence_df['Group'] = consequence_df['Group'].map({'G3': 'GroupN', 'G4': 'GroupAI'})

frequency_consequence = consequence_df.groupby('Group')['Consequence'].value_counts().unstack(). \
    sort_values(by='Group', ascending=False)

frequency_consequence.T.plot.bar()
plt.xlabel('Consequence')
plt.ylabel('Frequency')
plt.title('Frequency Plot')
plt.xticks(rotation=0)
plt.show()

'''Factor Analysis'''
# filtered_norm_mean_df = pd.read_excel('./merged_filtered_norm_mean_s1_2_3_data.xlsx')
# numeric_columns = filtered_norm_mean_df.select_dtypes(include=['number']).columns
#
# concatenated_df_fa = filtered_norm_mean_df[numeric_columns]
# fa = FactorAnalyzer(n_factors=4, rotation='varimax')
# fa.fit(concatenated_df_fa)
# eigen_values, vectors = fa.get_eigenvalues()
#
# plt.scatter(range(1, concatenated_df_fa.shape[1] + 1), eigen_values)
# plt.plot(range(1, concatenated_df_fa.shape[1] + 1), eigen_values)
# plt.title('Scree Plot')
# plt.xlabel('Factors')
# plt.ylabel('Eigenvalue')
# plt.grid()
# plt.show()
#
# fa.fit(concatenated_df_fa)
# fa = pd.DataFrame(fa.loadings_, index=concatenated_df_fa.columns)
# data = {
#     'index': fa.index,
#     'Factor1': fa.values[:, 0],
#     'Factor2': fa.values[:, 1],
#     'Factor3': fa.values[:, 2],
#     'Factor4': fa.values[:, 3]
# }
#
# df = pd.DataFrame(data)
# factor_indices = {f: [] for f in df.columns[1:]}
# for index, row in df.iterrows():
#     for factor in df.columns[1:]:
#         if row[factor] > 0.5 or row[factor] < -0.5:
#             factor_indices[factor].append(row['index'])
# for factor, indices in factor_indices.items():
#     print(f"{factor} indices:")
#     print(", ".join(indices))
# factors = json_normalize(factor_indices, max_level=2)
# print(factors)

# concatenated_df_fa = filtered_norm_mean_df[numeric_columns]
# if scenario == "1_2" or "3":
#     concatenated_df_fa.drop(columns=concatenated_df_fa.filter(regex='_overflow').columns, inplace=True)
# n_factors = 5 if scenario == "1_2_3" else 5
# fa = FactorAnalyzer(n_factors=10)
# fa.fit(concatenated_df_fa)
# eigen_values, vectors = fa.get_eigenvalues()
#
# # Plot the scree plot
# plt.scatter(range(1, concatenated_df_fa.shape[1] + 1), eigen_values)
# plt.plot(range(1, concatenated_df_fa.shape[1] + 1), eigen_values)
# plt.title('Scree Plot')
# plt.xlabel('Factors')
# plt.ylabel('Eigenvalue')
# plt.grid()
#
# # Find the elbow point
# cumulative_variance = np.cumsum(eigen_values) / np.sum(eigen_values)
# elbow_index = np.argmax(np.diff(cumulative_variance) < 0.05) + 1
#
# # Plot the elbow point as a dashed line and annotate the best factor
# plt.axvline(x=elbow_index, color='red', linestyle='--', label='Elbow')
# plt.annotate(f'Best Factor: {elbow_index}',
#              xy=(elbow_index, eigen_values[elbow_index - 1]),
#              xytext=(elbow_index + 0.5, eigen_values[elbow_index - 1] + 0.1),
#              arrowprops=dict(facecolor='black', arrowstyle='->'))
# plt.legend()
# plt.show()
#
# # Plot the cumulative variance explained
# plt.figure()
# plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o')
# plt.axvline(x=elbow_index, color='red', linestyle='--', label='Elbow')
# plt.annotate(f'Best Factor: {elbow_index}\nCumulative Variance: {cumulative_variance[elbow_index - 1]:.2%}',
#              xy=(elbow_index, cumulative_variance[elbow_index - 1]),
#              xytext=(elbow_index + 2, cumulative_variance[elbow_index - 1] + 0.02),
#              arrowprops=dict(facecolor='black', arrowstyle='->'))
# plt.title('Cumulative Variance Explained')
# plt.xlabel('Number of Factors')
# plt.ylabel('Cumulative Variance Explained')
# plt.grid()
# plt.legend()
# plt.show()
#
# # Best factor corresponding to the elbow point
# best_factor = elbow_index
# print(f"Best factor corresponding to the elbow point: {best_factor}")
#
# fa.fit(concatenated_df_fa)
# fa_loadings = pd.DataFrame(fa.loadings_, index=concatenated_df_fa.columns)
#
# data = {'index': fa_loadings.index}
# for i in range(1, best_factor + 1):
#     data[f'Factor{i}'] = fa_loadings.values[:, i - 1]
#
# df = pd.DataFrame(data)
#
# threshold = 0.4
# factor_indices = {f: [] for f in df.columns[1:]}
# for index, row in df.iterrows():
#     for factor in df.columns[1:]:
#         if abs(row[factor]) > threshold:
#             factor_indices[factor].append(row['index'])
#
# for factor, indices in factor_indices.items():
#     print(f"{factor} indices:")
#     print(", ".join(indices))
#
# factors = json_normalize(factor_indices, max_level=2)
# print(factors)

'''Radar Plot Within Participants'''
selected_participants = [
    "P021",
    "P024",
    # "P030",
    # "P032",
    # "P037",
    # "P096",
    # "P039",
]
columns = list(pd.read_excel(excel_file_path,
                             sheet_name=[
                                 'simulator_logs',
                                 'indexes',
                                 'support_questions',
                                 'watch_data',
                                 # 'eye_tracking',
                                 'AI_vs_human',
                                 # 'AI_questions',
                             ],
                             nrows=1).values())
flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
column_names = list(dict.fromkeys(flattened_list))
column_names = [column for column in column_names if column not in columns_to_filter]
filtered_df = filtered_norm_mean_df[column_names]
if scenario == "1_2" or "3":
    filtered_df.drop(columns=filtered_df.filter(regex='_overflow').columns, inplace=True)
numeric_columns = filtered_df.select_dtypes(include=['number']).columns
filtered_df = filtered_df[filtered_df['Group'].isin(['GroupAI'])]
filtered_df = filtered_df[filtered_df['Participant'].isin(selected_participants)]
groupby_participant = pd.pivot_table(filtered_df.drop(columns=['Group']), index=["Participant"], aggfunc="mean",
                                     sort=[False, False])
df_radar = pd.DataFrame(dict(
    value=groupby_participant.values.flatten(),
    variable=np.array([numeric_columns] * len(selected_participants)).flatten(),
    participant=np.array([[s_p] * len(numeric_columns) for s_p in selected_participants]).flatten(),
))
fig = px.line_polar(df_radar,
                    r='value',
                    theta='variable',
                    line_close=True,
                    color='participant',
                    )
fig.update_traces(fill='toself')
fig.update_layout(legend=dict(
    yanchor="top",
    y=1.15,
    xanchor="center",
    x=0.5
))
fig.update_layout(font=dict(size=14))
plotly.offline.plot(fig)

selected_participants = [
    # "P021",
    # "P024",
    "P030",
    "P032",
    # "P037",
    # "P096",
    # "P039",
]
columns = list(pd.read_excel(excel_file_path,
                             sheet_name=[
                                 'simulator_logs',
                                 'indexes',
                                 'support_questions',
                                 # 'watch_data',
                                 'eye_tracking',
                                 'AI_vs_human',
                                 'AI_questions',
                             ],
                             nrows=1).values())
flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
column_names = list(dict.fromkeys(flattened_list))
column_names = [column for column in column_names if column not in columns_to_filter]
filtered_df = filtered_norm_mean_df[column_names]
if scenario == "1_2" or "3":
    filtered_df.drop(columns=filtered_df.filter(regex='_overflow').columns, inplace=True)
numeric_columns = filtered_df.select_dtypes(include=['number']).columns
filtered_df = filtered_df[filtered_df['Group'].isin(['GroupAI'])]
filtered_df = filtered_df[filtered_df['Participant'].isin(selected_participants)]
groupby_participant = pd.pivot_table(filtered_df.drop(columns=['Group']), index=["Participant"], aggfunc="mean",
                                     sort=[False, False])

df_radar = pd.DataFrame(dict(
    value=groupby_participant.values.flatten(),
    variable=np.array([numeric_columns] * len(selected_participants)).flatten(),
    participant=np.array([[s_p] * len(numeric_columns) for s_p in selected_participants]).flatten(),
))
fig = px.line_polar(df_radar,
                    r='value',
                    theta='variable',
                    line_close=True,
                    color='participant',
                    )
fig.update_traces(fill='toself')
fig.update_layout(legend=dict(
    yanchor="top",
    y=1.15,
    xanchor="center",
    x=0.5
))
fig.update_layout(font=dict(size=14))
plotly.offline.plot(fig)

selected_participants = [
    # "P021",
    # "P024",
    # "P030",
    # "P032",
    "P037",
    "P096",
    # "P039",
]
columns = list(pd.read_excel(excel_file_path,
                             sheet_name=[
                                 'simulator_logs',
                                 'indexes',
                                 'support_questions',
                                 'watch_data',
                                 # 'eye_tracking',
                                 'AI_vs_human',
                                 'AI_questions',
                             ],
                             nrows=1).values())
flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
column_names = list(dict.fromkeys(flattened_list))
column_names = [column for column in column_names if column not in columns_to_filter]
filtered_df = filtered_norm_mean_df[column_names]
if scenario == "1_2" or "3":
    filtered_df.drop(columns=filtered_df.filter(regex='_overflow').columns, inplace=True)
numeric_columns = filtered_df.select_dtypes(include=['number']).columns
filtered_df = filtered_df[filtered_df['Group'].isin(['GroupAI'])]
filtered_df = filtered_df[filtered_df['Participant'].isin(selected_participants)]
groupby_participant = pd.pivot_table(filtered_df.drop(columns=['Group']), index=["Participant"], aggfunc="mean",
                                     sort=[False, False])
df_radar = pd.DataFrame(dict(
    value=groupby_participant.values.flatten(),
    variable=np.array([numeric_columns] * len(selected_participants)).flatten(),
    participant=np.array([[s_p] * len(numeric_columns) for s_p in selected_participants]).flatten(),
))
fig = px.line_polar(df_radar,
                    r='value',
                    theta='variable',
                    line_close=True,
                    color='participant',
                    )
fig.update_traces(fill='toself')
fig.update_layout(legend=dict(
    yanchor="top",
    y=1.15,
    xanchor="center",
    x=0.5
))
fig.update_layout(font=dict(size=14))
plotly.offline.plot(fig)

print('End of processing')
# For each part read the columns specified in the concatenated sheet and read the values of it from the merged_filtered_norm_mean_data
'''columns = list(pd.read_excel(excel_file_path,
                             sheet_name=[
                                 'simulator_logs',
                                 'indexes',
                                 'support_questions',
                                 # 'watch_data',
                                 # 'eye_tracking',
                                 # 'AI_vs_human',
                                 # 'AI_questions',
                             ],
                             nrows=1).values())

flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
column_names = list(dict.fromkeys(flattened_list))'''

'''**************************************************************************************************************'''
#  **************************************************************************************************************
'''**************************************************************************************************************'''
#
#                                              GroupN vs GroupAI

'''**************************************************************************************************************'''
'''**************************************************************************************************************'''

'''**************************************************************************************************************'''
#                                  Simulator Logs, Support Questions, Indexes
'''**************************************************************************************************************'''

# # """#**DATA PREPARATION**"""
# #
# columns = list(pd.read_excel(excel_file_path,
#                              sheet_name=[
#                                  'simulator_logs',
#                                  'indexes',
#                                  'support_questions',
#                                  # 'watch_data',
#                                  # 'eye_tracking',
#                                  # 'AI_vs_human',
#                                  # 'AI_questions',
#                              ],
#                              nrows=1).values())
#
# flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
# column_names = list(dict.fromkeys(flattened_list))
#
# filtered_df = filtered_norm_mean_df[column_names]
# print("missing data:\n", filtered_df.isnull().sum())
#
# nan_counts = filtered_df.isnull().sum(axis=1)
# filtered_df = filtered_df[nan_counts <= 5].reset_index(drop=True)
# print("missing data:\n", filtered_df.isnull().sum())
#
# filtered_df = filtered_df.dropna(thresh=len(filtered_df) - 4, axis=1)  # Drop columns with more than 20 missing values
# print("missing data:\n", filtered_df.isnull().sum())
#
# participants = filtered_df['Participant'].unique()
# print("Total Participants:", len(participants))
# #
# # '''*********************'''
# # Simulate missing values by replacing some values with NaN
# # np.random.seed(42)
# # test_impute_accuracy_df = filtered_df.dropna()
# # numeric_columns = test_impute_accuracy_df.select_dtypes(include=['number']).columns
# # missing_mask = np.random.rand(*test_impute_accuracy_df[numeric_columns].shape) < 0.2  # 20% missing values
# #
# # data_with_missing = test_impute_accuracy_df[numeric_columns].copy()
# # data_with_missing[missing_mask] = np.nan
# # ground_truth = test_impute_accuracy_df[numeric_columns].copy().to_numpy()
#
# # imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# # imputed_data = imputer.fit_transform(data_with_missing)
# # mae = mean_absolute_error(ground_truth[missing_mask], imputed_data[missing_mask])
# # print(f'Mean Absolute Error: {mae}')
# # print("missing data:\n", filtered_df.isnull().sum())
#
# '''*********************'''
# imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# numeric_columns = filtered_df.select_dtypes(include=['number']).columns
# imputed_df = filtered_df.copy()
# imputed_df[numeric_columns] = imputer.fit_transform(filtered_df[numeric_columns])
# print("missing data:\n", imputed_df.isnull().sum())
# imputed_df.to_excel('merged_imputed_logs_q_indx_data.xlsx', index=False)

'''**************************************************************************************************************'''

# imputed_df = pd.read_excel('merged_imputed_logs_q_indx_data.xlsx')
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
#
# '''**************************************************************************************************************'''
# '''**************************************************************************************************************'''
#
# """#**Group3 vs Group4**"""
# focused_df = imputed_df.copy()
# focused_df['Group'] = focused_df['Group'].map({'G3': 'GroupN', 'G4': 'GroupAI'})
# print("Participants Group3 vs Group4:", len(focused_df['Participant'].unique()))
#
# '''*********************'''
# focused_df_groups = focused_df.drop(columns=['Participant'])
# groupby_groups = pd.pivot_table(focused_df_groups, index=["Group"], aggfunc="mean", sort=[False, False]).\
#     sort_values(by='Group', ascending=False)
# groupby_groups.to_excel('./Paper/Images/recent_updated_plots/pivot_table_groups.xlsx')
#
# ax = groupby_groups.T.plot.bar()
# ax.set_title('Values for GroupN vs GroupAI and Metrics')
# ax.set_xlabel('Metric')
# plt.xticks(rotation=45)
# ax.set_ylabel('Mean Value')
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# correlation_groupby_groups = focused_df[numeric_columns].corr()
# matrix = np.triu(correlation_groupby_groups)
# plt.figure()
# sns.heatmap(correlation_groupby_groups, annot=True, cmap='coolwarm', fmt=".2f", mask=matrix)
# plt.title('Correlation Matrix')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# consequence_df = pd.read_excel('concatenated_data.xlsx', sheet_name='simulator_logs')
# consequence_df = consequence_df[consequence_df['Group'].isin(['G3', 'G4'])]
# consequence_df['Group'] = consequence_df['Group'].map({'G3': 'GroupN', 'G4': 'GroupAI'})
#
# frequency_consequence = consequence_df.groupby('Group')['Consequence'].value_counts().unstack().\
#     sort_values(by='Group', ascending=False)
#
# frequency_consequence.T.plot.bar()
# plt.xlabel('Consequence')
# plt.ylabel('Frequency')
# plt.title('Frequency Plot')
# plt.xticks(rotation=0)
# plt.show()
#
# '''*********************'''
# df = pd.DataFrame(dict(
#     value=groupby_groups.values.flatten(),
#     variable=np.array(([numeric_columns.values]*2)).flatten(),
#     Group=np.array([['GroupN']*len(numeric_columns), ['GroupAI']*len(numeric_columns)]).flatten()
# ))
# fig = px.line_polar(df,
#                     r='value',
#                     theta='variable',
#                     line_close=True,
#                     color='Group')
# fig.update_traces(fill='toself')
# plotly.offline.plot(fig)
#
# '''*********************'''
# melted_df = focused_df_groups.melt(
#     id_vars='Group',
#     value_vars=numeric_columns,
#     var_name='Variable',
#     value_name='Value').sort_values(by='Group', ascending=False)
#
# plt.figure()
# sns.boxplot(x='Variable', y='Value', data=melted_df, width=0.6, hue='Group')
# plt.xlabel('Metric')
# plt.ylabel('Value')
# plt.title('Boxplot Comparison Between Groups for Numeric Variables')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()

'''**************************************************************************************************************'''
#                               Simulator Logs, Support Questions, Indexes, Watch
'''**************************************************************************************************************'''

# """#**DATA PREPARATION**"""
#
# columns = list(pd.read_excel(excel_file_path,
#                              sheet_name=[
#                                  'simulator_logs',
#                                  'indexes',
#                                  'support_questions',
#                                  'watch_data',
#                                  # 'eye_tracking',
#                                  # 'AI_vs_human',
#                                  # 'AI_questions',
#                              ],
#                              nrows=1).values())
#
# flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
# column_names = list(dict.fromkeys(flattened_list))
#
# filtered_df = filtered_norm_mean_df[column_names]
# print("missing data:\n", filtered_df.isnull().sum())
#
# nan_counts = filtered_df.isnull().sum(axis=1)
# filtered_df = filtered_df[nan_counts < 3].reset_index(drop=True)
# print("missing data:\n", filtered_df.isnull().sum())
#
# filtered_df = filtered_df.dropna(thresh=len(filtered_df) - 4, axis=1)  # Drop columns with more than 20 missing values
# print("missing data:\n", filtered_df.isnull().sum())
#
# participants = filtered_df['Participant'].unique()
# print("Total Participants:", len(participants))

# '''*********************'''
# # Simulate missing values by replacing some values with NaN
# np.random.seed(42)
# test_impute_accuracy_df = filtered_df.dropna()
# numeric_columns = test_impute_accuracy_df.select_dtypes(include=['number']).columns
# missing_mask = np.random.rand(*test_impute_accuracy_df[numeric_columns].shape) < 0.2  # 20% missing values
#
# data_with_missing = test_impute_accuracy_df[numeric_columns].copy()
# data_with_missing[missing_mask] = np.nan
# ground_truth = test_impute_accuracy_df[numeric_columns].copy().to_numpy()
#
# imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# imputed_data = imputer.fit_transform(data_with_missing)
# mae = mean_absolute_error(ground_truth[missing_mask], imputed_data[missing_mask])
# print(f'Mean Absolute Error: {mae}')
# print("missing data:\n", filtered_df.isnull().sum())
#
# '''*********************'''
# imputed_df = filtered_df.copy()
# imputed_df[numeric_columns] = imputer.fit_transform(filtered_df[numeric_columns])
# print("missing data:\n", imputed_df.isnull().sum())
# imputed_df.to_excel('merged_imputed_logs_q_indx_watch_data.xlsx', index=False)

# '''**************************************************************************************************************'''

# imputed_df = pd.read_excel('merged_imputed_logs_q_indx_watch_data.xlsx')
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
# #
# # '''**************************************************************************************************************'''
# # '''**************************************************************************************************************'''
# #
# """#**Group3 vs Group4**"""
# focused_df = imputed_df.copy()
# focused_df['Group'] = focused_df['Group'].map({'G3': 'GroupN', 'G4': 'GroupAI'})
# print("Participants Group3 vs Group4:", len(focused_df['Participant'].unique()))
#
# concatenated_df_groups = focused_df.drop(columns=['Participant'])
#
# '''*********************'''
# groupby_groups = pd.pivot_table(concatenated_df_groups, index=["Group"], aggfunc="mean", sort=[False, False]).\
#     sort_values(by='Group', ascending=False)
# groupby_groups.to_excel('./Paper/Images/recent_updated_plots/pivot_table_with_watch_groups.xlsx')
#
# ax = groupby_groups.T.plot.bar()
# ax.set_title('Values for GroupN vs GroupAI and Metrics')
# ax.set_xlabel('Metric')
# plt.xticks(rotation=45)
# ax.set_ylabel('Mean Value')
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# correlation_groupby_groups = focused_df[numeric_columns].corr()
# matrix = np.triu(correlation_groupby_groups)
# plt.figure()
# sns.heatmap(correlation_groupby_groups, annot=True, cmap='coolwarm', fmt=".2f", mask=matrix)
# plt.title('Correlation Matrix')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# df = pd.DataFrame(dict(
#     value=groupby_groups.values.flatten(),
#     variable=np.array(([numeric_columns.values]*2)).flatten(),
#     Group=np.array([['GroupN']*len(numeric_columns), ['GroupAI']*len(numeric_columns)]).flatten()
# ))
# fig = px.line_polar(df,
#                     r='value',
#                     theta='variable',
#                     line_close=True,
#                     color='Group')
# fig.update_traces(fill='toself')
# plotly.offline.plot(fig)
#
# '''*********************'''
# melted_df = concatenated_df_groups.melt(
#     id_vars='Group',
#     value_vars=numeric_columns,
#     var_name='Variable',
#     value_name='Value').sort_values(by='Group', ascending=False)
#
# plt.figure()
# sns.boxplot(x='Variable', y='Value', data=melted_df, width=0.6, hue='Group')
# plt.xlabel('Metric')
# plt.ylabel('Value')
# plt.title('Boxplot Comparison Between Groups for Numeric Variables')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()

'''**************************************************************************************************************'''
#  **************************************************************************************************************
'''**************************************************************************************************************'''
#
#                                         GroupAI (Within Participants)

'''**************************************************************************************************************'''
'''**************************************************************************************************************'''

'''**************************************************************************************************************'''
#                                Simulator Logs, Support Questions, Indexes, AI
'''**************************************************************************************************************'''
#                                           Deep Reinforcement Learning
'''**************************************************************************************************************'''

# # """#**DATA PREPARATION**"""
# #
# columns = list(pd.read_excel(excel_file_path,
#                              sheet_name=[
#                                  'simulator_logs',
#                                  'indexes',
#                                  'support_questions',
#                                  # 'watch_data',
#                                  # 'eye_tracking',
#                                  'AI_vs_human',
#                                  # 'AI_questions',
#                              ],
#                              nrows=1).values())
#
# flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
# column_names = list(dict.fromkeys(flattened_list))
#
# filtered_df = filtered_norm_mean_df[column_names]
# filtered_df = filtered_df[filtered_df['Group'].isin(['G4'])]
# print("missing data:\n", filtered_df.isnull().sum())
#
# nan_counts = filtered_df.isnull().sum(axis=1)
# filtered_df = filtered_df[nan_counts < 5].reset_index(drop=True)
# print("missing data:\n", filtered_df.isnull().sum())
#
# filtered_df = filtered_df.dropna(thresh=len(filtered_df) - 0, axis=1)  # Drop columns with more than XX missing values
# print("missing data:\n", filtered_df.isnull().sum())
#
# participants = filtered_df['Participant'].unique()
# print("Total Participants:", len(participants))
# #
# # # '''*********************'''
# # # # Simulate missing values by replacing some values with NaN
# # # np.random.seed(42)
# # # test_impute_accuracy_df = filtered_df.dropna()
# # # numeric_columns = test_impute_accuracy_df.select_dtypes(include=['number']).columns
# # # missing_mask = np.random.rand(*test_impute_accuracy_df[numeric_columns].shape) < 0.2  # 20% missing values
# # #
# # # data_with_missing = test_impute_accuracy_df[numeric_columns].copy()
# # # data_with_missing[missing_mask] = np.nan
# # # ground_truth = test_impute_accuracy_df[numeric_columns].copy().to_numpy()
# # #
# # # imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# # # imputed_data = imputer.fit_transform(data_with_missing)
# # # mae = mean_absolute_error(ground_truth[missing_mask], imputed_data[missing_mask])
# # # print(f'Mean Absolute Error: {mae}')
# # # print("missing data:\n", filtered_df.isnull().sum())
# # #
# # # '''*********************'''
# imputed_df = filtered_df.copy()
# # # imputed_df[numeric_columns] = imputer.fit_transform(filtered_df[numeric_columns])
# # # print("missing data:\n", imputed_df.isnull().sum())
# imputed_df.to_excel('merged_with_drl_imputed_data.xlsx', index=False)
#
# '''**************************************************************************************************************'''
#
# imputed_df = pd.read_excel('merged_with_drl_imputed_data.xlsx')
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
# #
# # '''**************************************************************************************************************'''
# # '''**************************************************************************************************************'''
# #
# # """#**Group4**"""
# focused_df = imputed_df.copy()
# print("Participants Group4:", len(focused_df['Participant'].unique()))
#
# concatenated_df_participant = focused_df.drop(columns=['Group'])
#
# '''*********************'''
# groupby_participant = pd.pivot_table(concatenated_df_participant, index=["Participant"], aggfunc="mean", sort=[False, False])
# # df_ai = pd.read_excel(excel_file_path, sheet_name='AI_questions')
# # groupby_participant = pd.merge(groupby_participant, df_ai, on=['Participant'], how='outer')
# groupby_participant.to_excel('./Paper/Images/updated_plots/pivot_table_with_drl_participants.xlsx')
#
# selected_participants = ["P024", "P030", "P032"]
# filtered_df = concatenated_df_participant[concatenated_df_participant['Participant'].isin(selected_participants)]
# groupby_participant = pd.pivot_table(filtered_df, index=["Participant"], aggfunc="mean", sort=[False, False])
#
# ax = groupby_participant.T.plot.bar()
# ax.set_title('Values for GroupAI and Metrics')
# ax.set_xlabel('Metric')
# plt.xticks(rotation=45)
# ax.set_ylabel('Mean Value')
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# correlation_groupby_ai = focused_df[numeric_columns].corr()
# matrix = np.triu(correlation_groupby_ai)
# plt.figure()
# sns.heatmap(correlation_groupby_ai, annot=True, cmap='coolwarm', fmt=".2f", mask=matrix)
# plt.title('Correlation Matrix')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# selected_participants = ["P024", "P030", "P032"]
# filtered_df = concatenated_df_participant[concatenated_df_participant['Participant'].isin(selected_participants)]
#
# groupby_participant = pd.pivot_table(filtered_df, index=["Participant"], aggfunc="mean", sort=False)
#
# df_radar = pd.DataFrame(dict(
#     value=groupby_participant.values.flatten(),
#     variable=np.array([numeric_columns] * len(selected_participants)).flatten(),
#     participant=np.array([[s_p]*len(numeric_columns) for s_p in selected_participants]).flatten(),
# ))
# fig = px.line_polar(df_radar,
#                     r='value',
#                     theta='variable',
#                     line_close=True,
#                     color='participant'
#                     )
# fig.update_traces(fill='toself')
# plotly.offline.plot(fig)
#
# '''*********************'''
# focused_df = imputed_df
# concatenated_df_participant = focused_df['Participant'].unique()
# print("Participants Group4:", len(concatenated_df_participant))
# concatenated_df_participant = focused_df.drop(columns=['Group'])
#
# numeric_columns = concatenated_df_participant.select_dtypes(include=['number']).columns
# print("missing data:\n", concatenated_df_participant.isnull().sum())
# groupby_participant = pd.pivot_table(concatenated_df_participant, index=["Participant"], aggfunc="mean", sort=[False, False])
#
# melted_df = groupby_participant.melt(
#     value_vars=numeric_columns,
#     var_name='Variable',
#     value_name='Value')
# plt.figure()
# sns.boxplot(x='Variable', y='Value', data=melted_df, width=0.6)
# plt.xlabel('Metric')
# plt.ylabel('Value')
# plt.title('Boxplot Comparison Between Participant for Numeric Variables')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()

'''**************************************************************************************************************'''
#                                                AI Questions
''''''

# """#**DATA PREPARATION**"""
#
# columns = list(pd.read_excel(excel_file_path,
#                              sheet_name=[
#                                  'simulator_logs',
#                                  'indexes',
#                                  'support_questions',
#                                  # 'watch_data',
#                                  # 'eye_tracking',
#                                  'AI_vs_human',
#                                  'AI_questions',
#                              ],
#                              nrows=1).values())
#
# flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
# column_names = list(dict.fromkeys(flattened_list))
#
# filtered_df = filtered_norm_mean_df[column_names]
# filtered_df = filtered_df[filtered_df['Group'].isin(['G4'])]
# print("missing data:\n", filtered_df.isnull().sum())
#
# nan_counts = filtered_df.isnull().sum(axis=1)
# filtered_df = filtered_df[nan_counts < 5].reset_index(drop=True)
# print("missing data:\n", filtered_df.isnull().sum())
#
# filtered_df = filtered_df.dropna(thresh=len(filtered_df) - 1, axis=1)  # Drop columns with more than XX missing values
# print("missing data:\n", filtered_df.isnull().sum())
#
# participants = filtered_df['Participant'].unique()
# print("Total Participants:", len(participants))
#
# # '''*********************'''
# # # Simulate missing values by replacing some values with NaN
# # np.random.seed(42)
# # test_impute_accuracy_df = filtered_df.dropna()
# # numeric_columns = test_impute_accuracy_df.select_dtypes(include=['number']).columns
# # missing_mask = np.random.rand(*test_impute_accuracy_df[numeric_columns].shape) < 0.2  # 20% missing values
# #
# # data_with_missing = test_impute_accuracy_df[numeric_columns].copy()
# # data_with_missing[missing_mask] = np.nan
# # ground_truth = test_impute_accuracy_df[numeric_columns].copy().to_numpy()
# #
# # imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# # imputed_data = imputer.fit_transform(data_with_missing)
# # mae = mean_absolute_error(ground_truth[missing_mask], imputed_data[missing_mask])
# # print(f'Mean Absolute Error: {mae}')
# # print("missing data:\n", filtered_df.isnull().sum())
#
# '''*********************'''
# imputed_df = filtered_df.copy()
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
# imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# imputed_df[numeric_columns] = imputer.fit_transform(filtered_df[numeric_columns])
# print("missing data:\n", imputed_df.isnull().sum())
# imputed_df.to_excel('merged_imputed_logs_q_indx_drl_ai_data.xlsx', index=False)

'''**************************************************************************************************************'''

# imputed_df = pd.read_excel('merged_imputed_logs_q_indx_drl_ai_data.xlsx')
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
#
# # '''**************************************************************************************************************'''
# # '''**************************************************************************************************************'''
# #
# """#**Group4**"""
# focused_df = imputed_df.copy()
# concatenated_df_participant = focused_df.drop(columns=['Group'])
# print("Participants Group4:", len(concatenated_df_participant['Participant'].unique()))
#
# '''*********************'''
# groupby_participant = pd.pivot_table(concatenated_df_participant, index=["Participant"], aggfunc="mean", sort=[False, False]).\
#                       sort_values(by='Participant', ascending=False)
# groupby_participant.to_excel('./Paper/Images/recent_updated_plots/pivot_table_with_ai_participants.xlsx')
#
# selected_participants = ["P030", "P032"]
# filtered_df = concatenated_df_participant[concatenated_df_participant['Participant'].isin(selected_participants)]
# groupby_participant = pd.pivot_table(filtered_df, index=["Participant"], aggfunc="mean", sort=[False, False]).sort_values(by='Participant')
#
# ax = groupby_participant.T.plot.bar()
# ax.set_title('Values for GroupAI and Metrics')
# ax.set_xlabel('Metric')
# plt.xticks(rotation=45)
# ax.set_ylabel('Mean Value')
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# correlation_groupby_ai = focused_df[numeric_columns].corr()
# matrix = np.triu(correlation_groupby_ai)
# plt.figure(figsize=(40,40))
# sns.heatmap(correlation_groupby_ai, annot=True, cmap='coolwarm', fmt=".2f", mask=matrix)
# plt.title('Correlation Matrix')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# selected_participants = ["P030", "P032"]
# filtered_df = concatenated_df_participant[concatenated_df_participant['Participant'].isin(selected_participants)]
#
# groupby_participant = pd.pivot_table(filtered_df, index=["Participant"], aggfunc="mean", sort=[False, False])
#
# df_radar = pd.DataFrame(dict(
#     value=groupby_participant.values.flatten(),
#     variable=np.array([numeric_columns] * len(selected_participants)).flatten(),
#     participant=np.array([[s_p]*len(numeric_columns) for s_p in selected_participants]).flatten(),
# ))
#
# fig = px.line_polar(df_radar,
#                     r='value',
#                     theta='variable',
#                     line_close=True,
#                     color='participant'
#                     )
# fig.update_traces(fill='toself')
# plotly.offline.plot(fig)
#
# '''*********************'''
# melted_df = groupby_participant.melt(
#     value_vars=numeric_columns,
#     var_name='Variable',
#     value_name='Value')
# plt.figure()
# sns.boxplot(x='Variable', y='Value', data=melted_df, width=0.6)
# plt.xlabel('Metric')
# plt.ylabel('Value')
# plt.title('Boxplot Comparison Between Participant for Numeric Variables')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()

'''**************************************************************************************************************'''
#                             Simulator Logs, Support Questions, Indexes, AI, Watch
'''**************************************************************************************************************'''

# """#**DATA PREPARATION**"""
#
# columns = list(pd.read_excel(excel_file_path,
#                              sheet_name=[
#                                  'simulator_logs',
#                                  'indexes',
#                                  'support_questions',
#                                  # 'eye_tracking',
#                                  'AI_vs_human',
#                                  'AI_questions',
#                                  'watch_data',
#                              ],
#                              nrows=1).values())
#
# flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
# column_names = list(dict.fromkeys(flattened_list))
#
# filtered_df = filtered_norm_mean_df[column_names]
# filtered_df = filtered_df[filtered_df['Group'].isin(['G4'])]
# print("missing data:\n", filtered_df.isnull().sum())
#
# nan_counts = filtered_df.isnull().sum(axis=1)
# filtered_df = filtered_df[nan_counts < 3].reset_index(drop=True)
# print("missing data:\n", filtered_df.isnull().sum())
#
# filtered_df = filtered_df.dropna(thresh=len(filtered_df) - 1, axis=1)  # Drop columns with more than XX missing values
# print("missing data:\n", filtered_df.isnull().sum())
#
# participants = filtered_df['Participant'].unique()
# print("Total Participants:", len(participants))
# #
# # # '''*********************'''
# # # # Simulate missing values by replacing some values with NaN
# # # np.random.seed(42)
# # # test_impute_accuracy_df = filtered_df.dropna()
# # # numeric_columns = test_impute_accuracy_df.select_dtypes(include=['number']).columns
# # # missing_mask = np.random.rand(*test_impute_accuracy_df[numeric_columns].shape) < 0.2  # 20% missing values
# # #
# # # data_with_missing = test_impute_accuracy_df[numeric_columns].copy()
# # # data_with_missing[missing_mask] = np.nan
# # # ground_truth = test_impute_accuracy_df[numeric_columns].copy().to_numpy()
# # #
# # # imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# # # imputed_data = imputer.fit_transform(data_with_missing)
# # # mae = mean_absolute_error(ground_truth[missing_mask], imputed_data[missing_mask])
# # # print(f'Mean Absolute Error: {mae}')
# # # print("missing data:\n", filtered_df.isnull().sum())
# #
# # '''*********************'''
# imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# imputed_df = filtered_df.copy()
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
# imputed_df[numeric_columns] = imputer.fit_transform(filtered_df[numeric_columns])
# print("missing data:\n", imputed_df.isnull().sum())
# imputed_df.to_excel('merged_imputed_logs_q_indx_drl_ai_watch_data.xlsx', index=False)
#
# imputed_df = pd.read_excel('merged_imputed_logs_q_indx_drl_data.xlsx')
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
#
# # '''**************************************************************************************************************'''
# # '''**************************************************************************************************************'''
# #
# """#**Group4**"""
# focused_df = imputed_df.copy()
# concatenated_df_participant = focused_df.drop(columns=['Group'])
# print("Participants Group4:", len(concatenated_df_participant['Participant'].unique()))
#
# '''*********************'''
# groupby_participant = pd.pivot_table(concatenated_df_participant, index=["Participant"], aggfunc="mean", sort=[False, False]).\
#                       sort_values(by='Participant', ascending=False)
# groupby_participant.to_excel('./Paper/Images/recent_updated_plots/pivot_table_with_ai_participants.xlsx')
#
# selected_participants = ["P030", "P096"]
# filtered_df = concatenated_df_participant[concatenated_df_participant['Participant'].isin(selected_participants)]
# groupby_participant = pd.pivot_table(filtered_df, index=["Participant"], aggfunc="mean", sort=[False, False]).sort_values(by='Participant')
#
# ax = groupby_participant.T.plot.bar()
# ax.set_title('Values for GroupAI and Metrics')
# ax.set_xlabel('Metric')
# plt.xticks(rotation=45)
# ax.set_ylabel('Mean Value')
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# correlation_groupby_ai = focused_df[numeric_columns].corr()
# matrix = np.triu(correlation_groupby_ai)
# plt.figure(figsize=(40,40))
# sns.heatmap(correlation_groupby_ai, annot=True, cmap='coolwarm', fmt=".2f", mask=matrix)
# plt.title('Correlation Matrix')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# selected_participants = ["P030", "P096"]
# filtered_df = concatenated_df_participant[concatenated_df_participant['Participant'].isin(selected_participants)]
#
# groupby_participant = pd.pivot_table(filtered_df, index=["Participant"], aggfunc="mean", sort=[False, False])
#
# df_radar = pd.DataFrame(dict(
#     value=groupby_participant.values.flatten(),
#     variable=np.array([numeric_columns] * len(selected_participants)).flatten(),
#     participant=np.array([[s_p]*len(numeric_columns) for s_p in selected_participants]).flatten(),
# ))
#
# fig = px.line_polar(df_radar,
#                     r='value',
#                     theta='variable',
#                     line_close=True,
#                     color='participant'
#                     )
# fig.update_traces(fill='toself')
# plotly.offline.plot(fig)
#
# '''*********************'''
# melted_df = groupby_participant.melt(
#     value_vars=numeric_columns,
#     var_name='Variable',
#     value_name='Value')
# plt.figure()
# sns.boxplot(x='Variable', y='Value', data=melted_df, width=0.6)
# plt.xlabel('Metric')
# plt.ylabel('Value')
# plt.title('Boxplot Comparison Between Participant for Numeric Variables')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()


'''**************************************************************************************************************'''
#                         Simulator Logs, Support Questions, Indexes, AI, Watch, EyeTracker
'''**************************************************************************************************************'''

# """#**DATA PREPARATION**"""
#
# columns = list(pd.read_excel(excel_file_path,
#                              sheet_name=[
#                                  'simulator_logs',
#                                  'indexes',
#                                  'support_questions',
#                                  # 'eye_tracking',
#                                  'AI_vs_human',
#                                  'AI_questions',
#                                  'watch_data',
#                              ],
#                              nrows=1).values())
#
# flattened_list = ([item for sublist in columns for item in sublist if item not in ['Scenario']])
# column_names = list(dict.fromkeys(flattened_list))
#
# filtered_df = filtered_norm_mean_df[column_names]
# filtered_df = filtered_df[filtered_df['Group'].isin(['G4'])]
# print("missing data:\n", filtered_df.isnull().sum())
#
# nan_counts = filtered_df.isnull().sum(axis=1)
# filtered_df = filtered_df[nan_counts < 3].reset_index(drop=True)
# print("missing data:\n", filtered_df.isnull().sum())
#
# filtered_df = filtered_df.dropna(thresh=len(filtered_df) - 1, axis=1)  # Drop columns with more than XX missing values
# print("missing data:\n", filtered_df.isnull().sum())
#
# participants = filtered_df['Participant'].unique()
# print("Total Participants:", len(participants))
# #
# # # '''*********************'''
# # # # Simulate missing values by replacing some values with NaN
# # # np.random.seed(42)
# # # test_impute_accuracy_df = filtered_df.dropna()
# # # numeric_columns = test_impute_accuracy_df.select_dtypes(include=['number']).columns
# # # missing_mask = np.random.rand(*test_impute_accuracy_df[numeric_columns].shape) < 0.2  # 20% missing values
# # #
# # # data_with_missing = test_impute_accuracy_df[numeric_columns].copy()
# # # data_with_missing[missing_mask] = np.nan
# # # ground_truth = test_impute_accuracy_df[numeric_columns].copy().to_numpy()
# # #
# # # imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# # # imputed_data = imputer.fit_transform(data_with_missing)
# # # mae = mean_absolute_error(ground_truth[missing_mask], imputed_data[missing_mask])
# # # print(f'Mean Absolute Error: {mae}')
# # # print("missing data:\n", filtered_df.isnull().sum())
# #
# # '''*********************'''
# imputer = IterativeImputer(estimator=RandomForestRegressor(), max_iter=10, random_state=0, tol=1e-5, verbose=True)
# imputed_df = filtered_df.copy()
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
# imputed_df[numeric_columns] = imputer.fit_transform(filtered_df[numeric_columns])
# print("missing data:\n", imputed_df.isnull().sum())
# imputed_df.to_excel('merged_imputed_logs_q_indx_drl_ai_watch_data.xlsx', index=False)
#
# imputed_df = pd.read_excel('merged_imputed_logs_q_indx_drl_data.xlsx')
# numeric_columns = imputed_df.select_dtypes(include=['number']).columns
#
# # '''**************************************************************************************************************'''
# # '''**************************************************************************************************************'''
# #
# """#**Group4**"""
# focused_df = imputed_df.copy()
# concatenated_df_participant = focused_df.drop(columns=['Group'])
# print("Participants Group4:", len(concatenated_df_participant['Participant'].unique()))
#
# '''*********************'''
# groupby_participant = pd.pivot_table(concatenated_df_participant, index=["Participant"], aggfunc="mean", sort=[False, False]).\
#                       sort_values(by='Participant', ascending=False)
# groupby_participant.to_excel('./Paper/Images/recent_updated_plots/pivot_table_with_ai_participants.xlsx')
#
# selected_participants = ["P030", "P096"]
# filtered_df = concatenated_df_participant[concatenated_df_participant['Participant'].isin(selected_participants)]
# groupby_participant = pd.pivot_table(filtered_df, index=["Participant"], aggfunc="mean", sort=[False, False]).sort_values(by='Participant')
#
# ax = groupby_participant.T.plot.bar()
# ax.set_title('Values for GroupAI and Metrics')
# ax.set_xlabel('Metric')
# plt.xticks(rotation=45)
# ax.set_ylabel('Mean Value')
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# correlation_groupby_ai = focused_df[numeric_columns].corr()
# matrix = np.triu(correlation_groupby_ai)
# plt.figure(figsize=(40,40))
# sns.heatmap(correlation_groupby_ai, annot=True, cmap='coolwarm', fmt=".2f", mask=matrix)
# plt.title('Correlation Matrix')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()
#
# '''*********************'''
# selected_participants = ["P030", "P096"]
# filtered_df = concatenated_df_participant[concatenated_df_participant['Participant'].isin(selected_participants)]
#
# groupby_participant = pd.pivot_table(filtered_df, index=["Participant"], aggfunc="mean", sort=[False, False])
#
# df_radar = pd.DataFrame(dict(
#     value=groupby_participant.values.flatten(),
#     variable=np.array([numeric_columns] * len(selected_participants)).flatten(),
#     participant=np.array([[s_p]*len(numeric_columns) for s_p in selected_participants]).flatten(),
# ))
#
# fig = px.line_polar(df_radar,
#                     r='value',
#                     theta='variable',
#                     line_close=True,
#                     color='participant'
#                     )
# fig.update_traces(fill='toself')
# plotly.offline.plot(fig)
#
# '''*********************'''
# melted_df = groupby_participant.melt(
#     value_vars=numeric_columns,
#     var_name='Variable',
#     value_name='Value')
# plt.figure()
# sns.boxplot(x='Variable', y='Value', data=melted_df, width=0.6)
# plt.xlabel('Metric')
# plt.ylabel('Value')
# plt.title('Boxplot Comparison Between Participant for Numeric Variables')
# plt.xticks(rotation=45)
# plt.subplots_adjust(bottom=0.2)
# plt.show()

# """**DRL PREPARATION: AI vs Human Response (MSE)**"""

# concatenated_df_ai = pd.read_excel('concatenated_data.xlsx', sheet_name=['AI_questions', 'AI_vs_human'])['AI_questions']
# concatenated_df_drl = concatenated_df_ai.drop(focused_df[focused_df['Group'] == 'Group_3'].index)
# concatenated_df_drl.isnull().sum()
# concatenated_df_drl.drop(columns=['Alarm lists support'], inplace=True)
# concatenated_df_drl.isnull().sum()
#
# concatenated_df_drl[['AI vs human error', 'Consequence',
#                      'Recovery_Status', 'Recovery_time', 'Reaction_time', 'No_of_procedures',
#                      'Alarm_annunciations', 'Gaze point X', 'Gaze point Y',
#                      'Pupil position left X', 'Pupil position left Y',
#                      'Pupil position left Z', 'Pupil position right X',
#                      'Pupil position right Y', 'Pupil position right Z',
#                      'Pupil diameter left', 'Pupil diameter right',
#                      'Pupil diameter filtered', 'Fixation point X', 'Fixation point Y', 'SART_index', 'TLX_index',
#                      'SPAM_index',
#                      'Task Load', 'Alarm prioritization support',
#                      'Procedures support']] = concatenated_df_drl[['AI vs human error', 'Consequence',
#                                                                    'Recovery_Status', 'Recovery_time', 'Reaction_time',
#                                                                    'No_of_procedures',
#                                                                    'Alarm_annunciations', 'Gaze point X',
#                                                                    'Gaze point Y',
#                                                                    'Pupil position left X', 'Pupil position left Y',
#                                                                    'Pupil position left Z', 'Pupil position right X',
#                                                                    'Pupil position right Y', 'Pupil position right Z',
#                                                                    'Pupil diameter left', 'Pupil diameter right',
#                                                                    'Pupil diameter filtered', 'Fixation point X',
#                                                                    'Fixation point Y', 'SART_index', 'TLX_index',
#                                                                    'SPAM_index',
#                                                                    'Task Load', 'Alarm prioritization support',
#                                                                    'Procedures support']].astype(float)
#
# concatenated_df_drl = concatenated_df_drl.groupby(by=['Participant'], dropna=False).mean()
#
# """**DATA ANALYSIS: DRL(MSE)**"""
#
# concatenated_df_drl.dropna(inplace=True)
# scaler = preprocessing.MinMaxScaler()
#
# concatenated_df_drl.iloc[:, 0:] = scaler.fit_transform(concatenated_df_drl.iloc[:, 0:].to_numpy())
# concatenated_df_drl.corr()
#
# sns.heatmap(concatenated_df_drl.corr(), cmap='coolwarm')
# concatenated_df_drl.plot.bar()
#
# """**DATA PREPARATION : AI (DRL)**"""
#
# scaler2 = preprocessing.MinMaxScaler()
#
# concatenated_df_ai.iloc[:, 1:] = scaler2.fit_transform(concatenated_df_ai.iloc[:, 1:].to_numpy())
# concatenated_df_srla = pd.merge(concatenated_df_drl, concatenated_df_ai, on="Participant")
# concatenated_df_srla.dropna(inplace=True)
#
# df_watch = pd.read_excel('concatenated_data.xlsx', sheet_name=['watch_data'])['watch_data']
# df_watch.dropna(inplace=True)
# df_watch.groupby(['Group']).mean()
# df_watch = df_watch.groupby(['Participant']).mean()
#
# concatenated_df_srla_watch = pd.merge(df_watch, concatenated_df_srla, on="Participant")
# concatenated_df_srla.plot.bar()
#
# sns.set(font_scale=0.5)
# sns.heatmap(concatenated_df_srla.corr(), cmap='coolwarm')
#
# sns.set(font_scale=0.5)
# sns.heatmap(concatenated_df_srla_watch.corr(), cmap='coolwarm')

'''**************************************************************************************************************'''
#  **************************************************************************************************************
'''**************************************************************************************************************'''
#
#                                                  Factor Analysis

'''**************************************************************************************************************'''
'''**************************************************************************************************************'''

# filtered_norm_mean_df = pd.read_excel('./merged_filtered_norm_mean_s1_2_3_data.xlsx')
# numeric_columns = filtered_norm_mean_df.select_dtypes(include=['number']).columns
#
# concatenated_df_fa = filtered_norm_mean_df[numeric_columns]
#
# fa = FactorAnalyzer(n_factors=4, rotation='varimax')
#
# fa.fit(concatenated_df_fa)
#
# eigen_values, vectors = fa.get_eigenvalues()
#
# plt.scatter(range(1, concatenated_df_fa.shape[1] + 1), eigen_values)
# plt.plot(range(1, concatenated_df_fa.shape[1] + 1), eigen_values)
# plt.title('Scree Plot')
# plt.xlabel('Factors')
# plt.ylabel('Eigenvalue')
# plt.grid()
# plt.show()
#
# fa.fit(concatenated_df_fa)
# print(fa.get_factor_variance())
#
# fa = pd.DataFrame(fa.loadings_, index=concatenated_df_fa.columns)
#
# # Create a DataFrame from the provided data
# data = {
#     'index': fa.index,
#     'Factor1': fa.values[:, 0],
#     'Factor2': fa.values[:, 1],
#     'Factor3': fa.values[:, 2],
#     'Factor4': fa.values[:, 3]
# }
#
# df = pd.DataFrame(data)
#
# # Create empty dictionaries to store the indices for each factor
# factor_indices = {f: [] for f in df.columns[1:]}
#
# # Iterate through the DataFrame and add indices to the corresponding factor's list
# for index, row in df.iterrows():
#     for factor in df.columns[1:]:
#         if row[factor] > 0.5 or row[factor] < -0.5:
#             factor_indices[factor].append(row['index'])
#
# # Print the indices for each factor
# for factor, indices in factor_indices.items():
#     print(f"{factor} indices:")
#     print(", ".join(indices))
#
# # seaborn.pairplot(concatenated_df_srla, hue='AI vs human error')
# #
# # """# **DATA ANALYSIS (AI (DRL))**
# #
# # Group 3 vs Group4
# # """
# #
# # pd.pivot_table(concatenated_df_groups, index=["Group"], aggfunc="mean")
# #
# # """Group 4 Correlation Matrix (Within Participants)"""
# #
# # sns.set(font_scale=0.5)
# # sns.heatmap(concatenated_df_srla.corr(), cmap='coolwarm')
# #
# # sns.set(font_scale=0.5)
# # sns.heatmap(concatenated_df_srla_watch.corr(), cmap='coolwarm')
#
# """Factor Analysis (Group 4)"""
#
# json_normalize(factor_indices, max_level=2)
#
# """#**Pair Plot**"""
# seaborn.pairplot(imputed_df, hue='AI_vs_human_error')

"""# **Machine Learning**"""