The main objective of this analysis examine the liver dataset and mainly aim to cluster all the variable with similarity.
All the variables in the dataset are mainly associated with liver functions such as ‘TOTAL_BILIRUBIN','DIRECT_BILIRUBIN', 'ALKALINE_PHOSPHOTASE','ALAMINE_AMINOTRANSFERASE',
'ASPARTATE_AMINOTRANSFERASE','TOTAL_PROTIENS', 'ALBUMIN_AND_GLOBULIN_RATIO. Anther two variables are AGE and GENDER.
ALBUMIN is one the main test to assess liver function as decrease in ALBUMIN may suggest the imporper liver function. Therefore, I am also examine the whether the clusters in relation to the level of ALBUMIN is different for different clusters.
## Here are my python code for the data analysis
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 19 13:58:56 2021
"""
## K-mean Clustering Assigment
from pandas import Series, DataFrame
import pandas as pd
import numpy as np
import matplotlib.pylab as plt
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.cluster import KMeans
liver = pd.read_csv("liver.csv")
#upper-case all DataFrame column names
liver.columns = map(str.upper, liver.columns)
liver_clean = liver.dropna()
liver_clean["GENDER"]=liver_clean["GENDER"].replace("Male", "0")
liver_clean["GENDER"]=liver_clean["GENDER"].replace("Female", "1")
# subset clustering variables
cluster_liver=liver_clean[['AGE','GENDER','TOTAL_BILIRUBIN','DIRECT_BILIRUBIN',
'ALKALINE_PHOSPHOTASE','ALAMINE_AMINOTRANSFERASE',
'ASPARTATE_AMINOTRANSFERASE','TOTAL_PROTIENS',
'ALBUMIN_AND_GLOBULIN_RATIO']]
# standardize clustering variables to have mean=0 and sd=1
cluster_var=cluster_liver.copy()
cluster_var['AGE']=preprocessing.scale(cluster_var['AGE'].astype('float64'))
cluster_var['GENDER']=preprocessing.scale(cluster_var['GENDER'].astype('float64'))
cluster_var['TOTAL_BILIRUBIN']=preprocessing.scale(cluster_var['TOTAL_BILIRUBIN'].astype('float64'))
cluster_var['DIRECT_BILIRUBIN']=preprocessing.scale(cluster_var['DIRECT_BILIRUBIN'].astype('float64'))
cluster_var['ALKALINE_PHOSPHOTASE']=preprocessing.scale(cluster_var['ALKALINE_PHOSPHOTASE'].astype('float64'))
cluster_var['ALAMINE_AMINOTRANSFERASE']=preprocessing.scale(cluster_var['ALAMINE_AMINOTRANSFERASE'].astype('float64'))
cluster_var['ASPARTATE_AMINOTRANSFERASE']=preprocessing.scale(cluster_var['ASPARTATE_AMINOTRANSFERASE'].astype('float64'))
cluster_var['TOTAL_PROTIENS']=preprocessing.scale(cluster_var['TOTAL_PROTIENS'].astype('float64'))
cluster_var['ALBUMIN_AND_GLOBULIN_RATIO']=preprocessing.scale(cluster_var['ALBUMIN_AND_GLOBULIN_RATIO'].astype('float64'))
# split data into train and test sets
clus_train, clus_test = train_test_split(cluster_var, test_size=.3, random_state=123)
# k-means cluster analysis for 1-9 clusters
from scipy.spatial.distance import cdist
clusters=range(1,10)
meandist=[]
for k in clusters:
model=KMeans(n_clusters=k)
model.fit(clus_train)
clusassign=model.predict(clus_train)
meandist.append(sum(np.min(cdist(clus_train, model.cluster_centers_, 'euclidean'), axis=1))
/ clus_train.shape[0])
"""
Plot average distance from observations from the cluster centroid
to use the Elbow Method to identify number of clusters to choose
"""
plt.plot(clusters, meandist)
plt.xlabel('Number of clusters')
plt.ylabel('Average distance')
plt.title('Selecting k with the Elbow Method')
# Interpret 3 cluster solution
model3=KMeans(n_clusters=3)
model3.fit(clus_train)
clusassign=model3.predict(clus_train)
from sklearn.decomposition import PCA
pca_2 = PCA(2)
plot_columns = pca_2.fit_transform(clus_train)
plt.scatter(x=plot_columns[:,0], y=plot_columns[:,1], c=model3.labels_,)
plt.xlabel('Canonical variable 1')
plt.ylabel('Canonical variable 2')
plt.title('Scatterplot of Canonical Variables for 3 Clusters')
plt.show()
''' BEGIN multiple steps to merge cluster assignment with clustering variables to examine cluster variable means by cluster
'''
# create a unique identifier variable from the index for the
# cluster training data to merge with the cluster assignment variable
clus_train.reset_index(level=0, inplace=True)
# create a list that has the new index variable
cluslist=list(clus_train['index'])
# create a list of cluster assignments
labels=list(model3.labels_)
# combine index variable list with cluster assignment list into a dictionary
newlist=dict(zip(cluslist, labels))
newlist
# convert newlist dictionary to a dataframe
newclus=DataFrame.from_dict(newlist, orient='index')
newclus
# rename the cluster assignment column
newclus.columns = ['cluster']
# now do the same for the cluster assignment variable
# create a unique identifier variable from the index for the
# cluster assignment dataframe
# to merge with cluster training data
newclus.reset_index(level=0, inplace=True)
# merge the cluster assignment dataframe with the cluster training variable dataframe
# by the index variable
merged_train=pd.merge(clus_train, newclus, on='index')
merged_train.head(n=100)
# cluster frequencies
merged_train.cluster.value_counts()
"""
END multiple steps to merge cluster assignment with clustering variables to examine
cluster variable means by cluster
"""
# FINALLY calculate clustering variable means by cluster
clustergrp = merged_train.groupby('cluster').mean()
print ("Clustering variable means by cluster")
print(clustergrp)
# validate clusters in training data by examining cluster differences in GPA using ANOVA
# first have to merge GPA with clustering variables and cluster assignment data
albumin_data=liver_clean['ALBUMIN']
# split GPA data into train and test sets
albumin_train, albumin_test = train_test_split(albumin_data, test_size=.3, random_state=123)
albumin_train1=pd.DataFrame(albumin_train)
albumin_train1.reset_index(level=0, inplace=True)
merged_train_all=pd.merge(albumin_train1, merged_train, on='index')
sub1 = merged_train_all[['ALBUMIN', 'cluster']].dropna()
import statsmodels.formula.api as smf
import statsmodels.stats.multicomp as multi
albuminmod = smf.ols(formula='ALBUMIN ~ C(cluster)', data=sub1).fit()
print (albuminmod.summary())
print ('means for ALBUMIN by cluster')
m1= sub1.groupby('cluster').mean()
print (m1)
print ('standard deviations for ALBUMIN by cluster')
m2= sub1.groupby('cluster').std()
print (m2)
mc1 = multi.MultiComparison(sub1['ALBUMIN'], sub1['cluster'])
res1 = mc1.tukeyhsd()
print(res1.summary())
Three clusters have been found as shown in the plot.. In relation to ALBUMIN, the three clusters are significantly different using F test from the analysis of variance. Cluster (”0″) & Cluster(”2″) have similar mean, but both are different to cluster (”1″), which have lower mean.. That suggests that cluster (”2″) has lower level of protein and the liver function is poorer than clusters (”0″) and cluster (”3″).
