Using Kmeans to plot the silhoutte score for each feature

Question

Let me say I have a data of shape (100, 580, 10) where 100 is the different samples with 580 points and 10 features. I would like to use kmeans to find the silhouette score for all the 10 features across the 100 data samples and plot the silhouette score and the number of clusters.

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

# shape of feature vector
feature_vector = (100,580,10)

nr_clusters = range(2,10)
silhouette_avg = {}
# loop throth the data samples in the feature vector
for data_sample in range(len(feature_vector)):
    feature_data = feature_vector[data_sample]
    print(f'Here1'+ str(feature_data.shape))
    for feature in range(len(feature_vector[0])):
        silhouette_avg[spectra, feature] = []
        for num_clusters in nr_clusters:
            kmeans = KMeans(n_clusters=num_clusters, max_iter=50)labels = kmeans.labels_
            silhouette_avg[data_sample, feature].append(silhouette_score(feature_data, labels))
            print(f'Silhouette score for data sample {data_sample} and feature {feature} with {num_clusters} clusters is {silhouette_score(feature_data, labels)}')

    plt.plot(nr_clusters, silhouette_avg[data_sample, feature], label=f'Data sample {data_sample}')
    plt.ylabel('Silhouette Score')
    plt.xlabel('Number of Clusters')
    plt.title('Silhouette Score for each feature')
plt.legend()

I tried that but that rather gives me the silhouette score per data sample rather than the silhouette scores of all the 10 features across the 100 data samples.

Answer 1

I have converted my feature_vector to a pandas dataframe and then use that dataframe to loop through the number of columns (as feature vectors) and assign those in a dictionary.

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

# shape of feature vector
feature_vector = (100,580,10)

nr_clusters = range(2,10)
silhouette_avg = {}
# loop throth the data samples in the feature vector
for feature in feature_vector.columns:
    silhouette_avg[feature] = []
    feature_data = feature_vector[feature].values.reshape(-1, 1)    
    for num_clusters in nr_clusters:
        kmeans = KMeans(n_clusters=num_clusters, max_iter=50)
        labels = kmeans.labels_
        silhouette_avg[feature].append(silhouette_score(feature_data,labels))
        print(f'Silhouette score for feature {feature} with {num_clusters} clusters is {silhouette_score(feature_data, labels)}')

fig, ax = plt.subplots(figsize=(10, 6))
ax.plot(nr_clusters, silhouette_avg['feature_vec1'],linestyle='-')
ax.plot(nr_clusters, silhouette_avg['feature_vec2'], linestyle='--')
ax.plot(nr_clusters, silhouette_avg['feature_vec3'], linestyle=':')
ax.plot(nr_clusters, silhouette_avg['feature_vec4'], linestyle='-.')
ax.plot(nr_clusters, silhouette_avg['feature_vec5'], linestyle=(0, (1, 10)))
ax.plot(nr_clusters, silhouette_avg['feature_vec6'], linestyle=(0, (1, 1)))
ax.plot(nr_clusters, silhouette_avg['feature_vec7'], linestyle=(0, (5, 10)))
ax.plot(nr_clusters, silhouette_avg['feature_vec8'], linestyle=(0, (5, 1)))
ax.plot(nr_clusters, silhouette_avg['feature_vec9'], linestyle=(0, (3, 10, 1, 10)))
ax.plot(nr_clusters, silhouette_avg['feature_vec10'], linestyle=(0, (3, 1, 1, 1)))   
ax.set_xlabel('Number of clusters')
ax.set_ylabel('Silhouette score')   
ax.set_title('Silhouette score for each feature')
ax.legend(['feature_vec1', 'feature_vec2', 'feature_vec3', 'feature_vec4', 'feature_vec5', 'feature_vec6', 'feature_vec7', 'feature_vec8', 'feature_vec9', 'feature_vec10'], loc='best')
plt.legend()
plt.show()