In [2]:
! pip install imagehash
import pandas as pd 
import os

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import glob
import matplotlib.pyplot as plt
import imagehash
import psutil

from PIL import Image
from joblib import Parallel, delayed

import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from IPython import display
import time

%matplotlib inline

plt.style.use('ggplot')
pd.set_option('display.max_columns', 300)
pd.set_option('display.max_rows', 100)
Collecting imagehash
  Downloading https://files.pythonhosted.org/packages/58/47/e639093d67f54e59163f9d259575a10bd16e4d0ff3bb1259ba64b9c77a42/ImageHash-4.0-py2.py3-none-any.whl (295kB)
    100% |████████████████████████████████| 296kB 1.3MB/s ta 0:00:011
Requirement already satisfied: pywavelets in /opt/conda/lib/python3.6/site-packages (from imagehash)
Requirement already satisfied: scipy in /opt/conda/lib/python3.6/site-packages (from imagehash)
Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from imagehash)
Requirement already satisfied: pillow in /opt/conda/lib/python3.6/site-packages (from imagehash)
Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from imagehash)
Installing collected packages: imagehash
Successfully installed imagehash-4.0

Load CSV

In [6]:
from clsdataset import * 

data_set = 'ISIC2019'  # retino / seeds / hist
data_dir = '/root/sharedfolder/production/DATA/' + data_set + "/"
train_dir = data_dir + 'ISIC_2019_Training_Input/'
use_gpu = torch.cuda.is_available()

classes, class_to_idx, idx_to_class, df=None,None,None,None

if data_set=='ISIC2019':                        
    classes, class_to_idx, idx_to_class, df = GenericDataset.find_classes_melanoma(
        train_dir,
        csv_file="/root/sharedfolder/production/DATA/ISIC2019/ISIC_2019_Training_GroundTruth.csv"
    )    

print(classes)
print(class_to_idx)
print(idx_to_class)
    
train = df

train.head()
Sorted Classes: ['MEL', 'NV', 'BCC', 'AK', 'BKL', 'DF', 'VASC', 'SCC', 'UNK']
class_to_idx: {'MEL': 0, 'NV': 1, 'BCC': 2, 'AK': 3, 'BKL': 4, 'DF': 5, 'VASC': 6, 'SCC': 7, 'UNK': 8}
num_to_class: {0: 'MEL', 1: 'NV', 2: 'BCC', 3: 'AK', 4: 'BKL', 5: 'DF', 6: 'VASC', 7: 'SCC', 8: 'UNK'}
['MEL', 'NV', 'BCC', 'AK', 'BKL', 'DF', 'VASC', 'SCC', 'UNK']
{'MEL': 0, 'NV': 1, 'BCC': 2, 'AK': 3, 'BKL': 4, 'DF': 5, 'VASC': 6, 'SCC': 7, 'UNK': 8}
{0: 'MEL', 1: 'NV', 2: 'BCC', 3: 'AK', 4: 'BKL', 5: 'DF', 6: 'VASC', 7: 'SCC', 8: 'UNK'}
Out[6]:
file category category_id
0 /root/sharedfolder/production/DATA/ISIC2019/IS... NV 1
1 /root/sharedfolder/production/DATA/ISIC2019/IS... NV 1
2 /root/sharedfolder/production/DATA/ISIC2019/IS... MEL 0
3 /root/sharedfolder/production/DATA/ISIC2019/IS... NV 1
4 /root/sharedfolder/production/DATA/ISIC2019/IS... MEL 0

Calculating the Hash, Shape, Mode, Length and Ratio of each image

In [13]:
import glob 

def getImageMetaData(file_path):
    with Image.open(file_path) as img:
        img_hash = imagehash.phash(img)
        return img.size, img.mode, str(img_hash), file_path

    
img_meta_l = Parallel(n_jobs=psutil.cpu_count(), verbose=1)(
    (delayed(getImageMetaData)(fp) for fp in glob.glob('/root/sharedfolder/production/DATA/ISIC2019/**/*.jpg',recursive=True))
)
img_meta_df = pd.DataFrame(np.array(img_meta_l))
img_meta_df.columns = ['Size', 'Mode', 'Hash', 'path']
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.
[Parallel(n_jobs=10)]: Done  30 tasks      | elapsed:    1.9s
[Parallel(n_jobs=10)]: Done 180 tasks      | elapsed:    4.4s
[Parallel(n_jobs=10)]: Done 596 tasks      | elapsed:   10.6s
[Parallel(n_jobs=10)]: Done 1296 tasks      | elapsed:   21.9s
[Parallel(n_jobs=10)]: Done 2196 tasks      | elapsed:   35.6s
[Parallel(n_jobs=10)]: Done 3296 tasks      | elapsed:   53.4s
[Parallel(n_jobs=10)]: Done 4596 tasks      | elapsed:  1.2min
[Parallel(n_jobs=10)]: Done 6096 tasks      | elapsed:  1.6min
[Parallel(n_jobs=10)]: Done 7796 tasks      | elapsed:  2.1min
[Parallel(n_jobs=10)]: Done 9696 tasks      | elapsed:  2.6min
[Parallel(n_jobs=10)]: Done 11796 tasks      | elapsed:  3.2min
[Parallel(n_jobs=10)]: Done 14096 tasks      | elapsed:  3.8min
[Parallel(n_jobs=10)]: Done 16596 tasks      | elapsed:  4.4min
[Parallel(n_jobs=10)]: Done 19296 tasks      | elapsed:  5.0min
[Parallel(n_jobs=10)]: Done 22196 tasks      | elapsed:  5.8min
[Parallel(n_jobs=10)]: Done 25312 out of 25331 | elapsed:  6.4min remaining:    0.3s
[Parallel(n_jobs=10)]: Done 25331 out of 25331 | elapsed:  6.4min finished
In [14]:
img_meta_df.columns = ['Size', 'Mode', 'Hash', 'file']
df = df.merge(img_meta_df, on='file', how='left')
df.head()
Out[14]:
file category category_id Size Mode Hash
0 /root/sharedfolder/production/DATA/ISIC2019/IS... NV 1 (1022, 767) RGB ad61c01d9f9a3ac3
1 /root/sharedfolder/production/DATA/ISIC2019/IS... NV 1 (1022, 767) RGB b331cccc333389cb
2 /root/sharedfolder/production/DATA/ISIC2019/IS... MEL 0 (1022, 767) RGB a093876d4537136f
3 /root/sharedfolder/production/DATA/ISIC2019/IS... NV 1 (1022, 767) RGB e1709694cd63c9cd
4 /root/sharedfolder/production/DATA/ISIC2019/IS... MEL 0 (1022, 767) RGB 966d693231936793
In [15]:
df.to_csv('./image_info.csv', index=False)
In [16]:
df_gb = df.groupby('Hash').count().reset_index()
df_gb.head()
Out[16]:
Hash file category category_id Size Mode
0 800048c8ffefdead 1 1 1 1 1
1 8001091777dddd77 1 1 1 1 1
2 80020dbf27da5dbd 1 1 1 1 1
3 800251d95767577f 1 1 1 1 1
4 800281af8d7757bf 1 1 1 1 1
In [18]:
df_gb_dup = df_gb.query('file > 1')
df_gb_dup
Out[18]:
Hash file category category_id Size Mode
74 80375f7a7a6468e1 2 2 2 2 2
175 807b455f55a644f9 2 2 2 2 2
499 8197011705df177f 2 2 2 2 2
545 81c25e953b9e2cda 2 2 2 2 2
1537 86dc996766993364 2 2 2 2 2
1652 8a7fdd9dc8d8d080 2 2 2 2 2
1924 90c24e953ece689f 2 2 2 2 2
1949 90cf457f108f133d 2 2 2 2 2
2060 91274c0e757145bf 2 2 2 2 2
2182 914f443353ce4e73 2 2 2 2 2
2234 915f447351ce5473 2 2 2 2 2
2333 918d4c7375cad4d8 2 2 2 2 2
2929 943d51d3444e52fb 2 2 2 2 2
2976 94646bc76d9a3469 2 2 2 2 2
3039 948f557755c151d1 2 2 2 2 2
3257 951b55b5514f5417 2 2 2 2 2
3284 951f4ae02fc23da5 2 2 2 2 2
3390 953c5a632ed27929 2 2 2 2 2
3492 95584a352f367de0 2 2 2 2 2
3654 95914aca3bca3d4d 2 2 2 2 2
3733 9599546643ce4e1f 2 2 2 2 2
3852 95c64a3c6bc33dc8 2 2 2 2 2
3926 95dc5173463e4ae8 2 2 2 2 2
4270 97952ebc32615e8c 2 2 2 2 2
4498 a005957a7bed86d2 2 2 2 2 2
5274 a49399f2678cd4d2 2 2 2 2 2
5683 a5d595519553954b 2 2 2 2 2
5729 a607998e66a699cb 2 2 2 2 2
5822 a659991d6673998c 2 2 2 2 2
6006 a6cc993366ccb391 2 2 2 2 2
6020 a6d199ce64997631 2 2 2 2 2
6961 af3ee1519a8f3450 2 2 2 2 2
7015 af99c46ba4731b28 2 2 2 2 2
7080 b007d57c64c75d52 2 2 2 2 2
7371 b067c19b056c4777 2 2 2 2 2
7377 b067d15345534757 2 2 2 2 2
7506 b096cf493c26c3d9 2 2 2 2 2
7656 b0c34d3a36e25dcc 2 2 2 2 2
8074 b131cece3131ccce 2 2 2 2 2
8084 b131cecf87311ca6 2 2 2 2 2
8760 b1d3cf3cc63230c3 2 2 2 2 2
9619 b387c5d51495451f 2 2 2 2 2
10279 b43c998971e64177 2 2 2 2 2
10365 b463c2cc4fd0385f 2 2 2 2 2
10803 b4d9cb64309ae3c9 2 2 2 2 2
11776 b891800f4797177f 2 2 2 2 2
12445 bcb2c2cdc28bc36c 2 2 2 2 2
12653 bd8cc761d7848c63 2 2 2 2 2
13378 c03f2ff43dc2308b 2 2 2 2 2
13891 c21320dfc5ab0bf6 2 2 2 2 2
... ... ... ... ... ... ...
15011 c5ef145c518f138b 2 2 2 2 2
15096 c6672bb034c3b0cf 2 2 2 2 2
15297 c8a979c267e48667 2 2 2 2 2
15831 d0cf4530148f577e 2 2 2 2 2
15982 d1330a712fc53cce 2 2 2 2 2
16019 d13b02ed4ab02f79 2 2 2 2 2
16854 d4360a332fcc3d33 2 2 2 2 2
17048 d4914ac22f0f3dcd 2 2 2 2 2
17251 d5094a7e2f316f30 2 2 2 2 2
17321 d513515e52ab4a6e 2 2 2 2 2
17378 d51b503b0a6d1a7d 2 2 2 2 2
17410 d51f4063129f0b3f 2 2 2 2 2
17452 d52b0a3d2f743d30 2 2 2 2 2
17550 d53b4ae32fd03d10 2 2 2 2 2
17987 d5944ac82f6b3c65 2 2 2 2 2
18415 d69036624efb36a4 2 2 2 2 2
18708 e06dce9698d9611e 2 2 2 2 2
18895 e13e96c3c91c33d1 2 2 2 2 2
18913 e14d967259ce6649 2 2 2 2 2
19273 e1cf9e3061ce309d 2 2 2 2 2
19280 e1d78679119312de 2 2 2 2 2
19374 e2811537413f5d4f 2 2 2 2 2
20401 e56c9a9221cfe64c 2 2 2 2 2
20622 e5b496d219d90b69 2 2 2 2 2
21010 e68289d96666dcca 2 2 2 2 2
21087 e699996666999926 2 2 2 2 2
21470 e7999a66c9647162 2 2 2 2 2
21589 e84985bdc93ac93a 2 2 2 2 2
21762 e949d36ec19c94c5 2 2 2 2 2
21864 e9ca86cd9a93961c 2 2 2 2 2
22012 ea91aac41f16f9c2 2 2 2 2 2
22056 eab5bd17b4968032 2 2 2 2 2
22389 ed63d0b48fca18c9 2 2 2 2 2
22654 ef80c6195b39c1f4 2 2 2 2 2
23029 f1528547944f07de 2 2 2 2 2
23085 f179ce92ca1c196c 2 2 2 2 2
23105 f18cce33386339cc 2 2 2 2 2
23311 f248cd3076cd9972 2 2 2 2 2
23424 f2eccd34d06bb680 2 2 2 2 2
23511 f344cc3032cfc973 2 2 2 2 2
23534 f362cc893262cdd9 2 2 2 2 2
23918 f47055c7417155d6 2 2 2 2 2
23932 f47c80b8d5a3c1a7 2 2 2 2 2
24135 f552d76a505a42cb 2 2 2 2 2
24443 f72855626689dd26 2 2 2 2 2
24795 fa85c458873a7ca5 2 2 2 2 2
24817 fab1c1689ee68343 2 2 2 2 2
24955 fc75944b13354f50 2 2 2 2 2
25042 fde1c21ed812e193 3 3 3 3 3
25201 ffc1851c49d2a639 2 2 2 2 2

104 rows × 6 columns

In [19]:
df_gb_dup['file'].value_counts()
Out[19]:
2    103
3      1
Name: file, dtype: int64
In [20]:
dup_hash_l = df_gb_dup['Hash'].values
In [21]:
df_dup = df.loc[df['Hash'].isin(dup_hash_l)].sort_values('Hash')
df_dup.to_csv('./duplicated_info.csv')
df_dup.head(10)
Out[21]:
file category category_id Size Mode Hash
20563 /root/sharedfolder/production/DATA/ISIC2019/IS... BCC 2 (1024, 1024) RGB 80375f7a7a6468e1
17216 /root/sharedfolder/production/DATA/ISIC2019/IS... BCC 2 (1024, 1024) RGB 80375f7a7a6468e1
22221 /root/sharedfolder/production/DATA/ISIC2019/IS... MEL 0 (1024, 1024) RGB 807b455f55a644f9
14935 /root/sharedfolder/production/DATA/ISIC2019/IS... MEL 0 (1024, 1024) RGB 807b455f55a644f9
21402 /root/sharedfolder/production/DATA/ISIC2019/IS... AK 3 (1024, 1024) RGB 8197011705df177f
17878 /root/sharedfolder/production/DATA/ISIC2019/IS... AK 3 (1024, 1024) RGB 8197011705df177f
21878 /root/sharedfolder/production/DATA/ISIC2019/IS... AK 3 (1024, 1024) RGB 81c25e953b9e2cda
24025 /root/sharedfolder/production/DATA/ISIC2019/IS... AK 3 (1024, 1024) RGB 81c25e953b9e2cda
8676 /root/sharedfolder/production/DATA/ISIC2019/IS... NV 1 (600, 450) RGB 86dc996766993364
3505 /root/sharedfolder/production/DATA/ISIC2019/IS... NV 1 (600, 450) RGB 86dc996766993364
In [22]:
samp_hash = df_dup['Hash'].sample(1).values[0]
print(samp_hash)
b067c19b056c4777
In [23]:
dups = df_dup.query('Hash == @samp_hash')['file'].values

fig, ax = plt.subplots(len(dups), 1, figsize=(7, 5 * len(dups)))
for i, d in enumerate(dups):
    ax[i].imshow(mpimg.imread(d))
    ax[i].grid(alpha=0.1)
fig.tight_layout();
In [25]:
dups = df_dup.query('Hash == "b067c19b056c4777"')['file'].values

fig, ax = plt.subplots(len(dups), 1, figsize=(7, 5 * len(dups)))
for i, d in enumerate(dups):
    ax[i].imshow(mpimg.imread(d))
    ax[i].grid(alpha=0.1)
fig.tight_layout();

EOF

In [ ]: