! pip install imagehash
import pandas as pd
import os
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import glob
import matplotlib.pyplot as plt
import imagehash
import psutil
from PIL import Image
from joblib import Parallel, delayed
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from IPython import display
import time
%matplotlib inline
plt.style.use('ggplot')
pd.set_option('display.max_columns', 300)
pd.set_option('display.max_rows', 100)
from clsdataset import *
data_set = 'ISIC2019' # retino / seeds / hist
data_dir = '/root/sharedfolder/production/DATA/' + data_set + "/"
train_dir = data_dir + 'ISIC_2019_Training_Input/'
use_gpu = torch.cuda.is_available()
classes, class_to_idx, idx_to_class, df=None,None,None,None
if data_set=='ISIC2019':
classes, class_to_idx, idx_to_class, df = GenericDataset.find_classes_melanoma(
train_dir,
csv_file="/root/sharedfolder/production/DATA/ISIC2019/ISIC_2019_Training_GroundTruth.csv"
)
print(classes)
print(class_to_idx)
print(idx_to_class)
train = df
train.head()
Calculating the Hash, Shape, Mode, Length and Ratio of each image
import glob
def getImageMetaData(file_path):
with Image.open(file_path) as img:
img_hash = imagehash.phash(img)
return img.size, img.mode, str(img_hash), file_path
img_meta_l = Parallel(n_jobs=psutil.cpu_count(), verbose=1)(
(delayed(getImageMetaData)(fp) for fp in glob.glob('/root/sharedfolder/production/DATA/ISIC2019/**/*.jpg',recursive=True))
)
img_meta_df = pd.DataFrame(np.array(img_meta_l))
img_meta_df.columns = ['Size', 'Mode', 'Hash', 'path']
img_meta_df.columns = ['Size', 'Mode', 'Hash', 'file']
df = df.merge(img_meta_df, on='file', how='left')
df.head()
df.to_csv('./image_info.csv', index=False)
df_gb = df.groupby('Hash').count().reset_index()
df_gb.head()
df_gb_dup = df_gb.query('file > 1')
df_gb_dup
df_gb_dup['file'].value_counts()
dup_hash_l = df_gb_dup['Hash'].values
df_dup = df.loc[df['Hash'].isin(dup_hash_l)].sort_values('Hash')
df_dup.to_csv('./duplicated_info.csv')
df_dup.head(10)
samp_hash = df_dup['Hash'].sample(1).values[0]
print(samp_hash)
dups = df_dup.query('Hash == @samp_hash')['file'].values
fig, ax = plt.subplots(len(dups), 1, figsize=(7, 5 * len(dups)))
for i, d in enumerate(dups):
ax[i].imshow(mpimg.imread(d))
ax[i].grid(alpha=0.1)
fig.tight_layout();
dups = df_dup.query('Hash == "b067c19b056c4777"')['file'].values
fig, ax = plt.subplots(len(dups), 1, figsize=(7, 5 * len(dups)))
for i, d in enumerate(dups):
ax[i].imshow(mpimg.imread(d))
ax[i].grid(alpha=0.1)
fig.tight_layout();