{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix, precision_recall_curve\n",
    "\n",
    "base_dirs = ['./2nd_Battery/unet',\n",
    "            './2nd_Battery/unet-dice-loss',\n",
    "            './2nd_Battery/unet-focal-loss',\n",
    "            './2nd_Battery/unet-l1',\n",
    "            './2nd_Battery/unet-l2']\n",
    "colors = ['red', 'orange', 'yellow', 'pink', 'green', 'gold', 'magenta', 'cyan', 'violet']\n",
    "losses = [0.2072,  0.3879, 0.0112, 0.0357, 0.0241]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unet - precision: 0.582, recall: 0.895, accuracy: 0.958, f1: 0.685, iou: 0.550\n",
      "unet-dice-loss - precision: 0.509, recall: 0.934, accuracy: 0.946, f1: 0.642, iou: 0.492\n",
      "unet-focal-loss - precision: 0.894, recall: 0.762, accuracy: 0.984, f1: 0.818, iou: 0.711\n",
      "unet-l1 - precision: 0.670, recall: 0.885, accuracy: 0.971, f1: 0.752, iou: 0.618\n",
      "unet-l2 - precision: 0.778, recall: 0.788, accuracy: 0.977, f1: 0.777, iou: 0.649\n"
     ]
    }
   ],
   "source": [
    "lst_rst, lst_cdata = [], []\n",
    "idx = 0\n",
    "\n",
    "for base_dir in base_dirs:\n",
    "    result_dir = os.path.join(base_dir, 'result')\n",
    "\n",
    "    ##\n",
    "    lst_data = os.listdir(os.path.join(result_dir, 'numpy'))\n",
    "\n",
    "    lst_img = [f for f in lst_data if f.startswith('img')]\n",
    "    lst_gt = [f for f in lst_data if f.startswith('gt')]\n",
    "    lst_pr = [f for f in lst_data if f.startswith('pr')]\n",
    "\n",
    "    lst_img.sort()\n",
    "    lst_gt.sort()\n",
    "    lst_pr.sort()\n",
    "\n",
    "    ##\n",
    "    # id = 0\n",
    "    length = 1000\n",
    "    tp_list, tn_list, fp_list, fn_list, rst_list = [], [], [], [], []\n",
    "\n",
    "    for id in range(0, length):\n",
    "        img = np.load(os.path.join(result_dir,\"numpy\", lst_img[id]))\n",
    "        gt = np.load(os.path.join(result_dir,\"numpy\", lst_gt[id]))\n",
    "        pr = np.load(os.path.join(result_dir,\"numpy\", lst_pr[id]))\n",
    "\n",
    "        tp = np.sum(np.logical_and(gt == 1, pr == 1))\n",
    "        tn = np.sum(np.logical_and(gt == 0, pr == 0))\n",
    "        fp = np.sum(np.logical_and(gt == 0, pr == 1))\n",
    "        fn = np.sum(np.logical_and(gt == 1, pr == 0))\n",
    "\n",
    "        precision = tp / (tp + fp)  # precision = TP / (TP + FP)\n",
    "        recall = tp / (tp + fn) # recall = TP / (TP + FN), SE(Sensitivity), hit rate\n",
    "        accuracy = (tp + tn) / (tp + tn + fp + fn)\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "        iou = tp / (tp + fn + fp) \n",
    "\n",
    "        tp_list.append(tp)\n",
    "        tn_list.append(tn)\n",
    "        fp_list.append(fp)\n",
    "        fn_list.append(fn)\n",
    "\n",
    "\n",
    "        # output_binary = (pr > 0).astype(np.int32)\n",
    "        # label_flat = gt.flatten().astype(np.int32)\n",
    "        # output_flat = output_binary.flatten().astype(np.int32)\n",
    "\n",
    "        # accuracy = accuracy_score(label_flat, output_flat)\n",
    "        # precision = precision_score(label_flat, output_flat)\n",
    "        # recall = recall_score(label_flat, output_flat)\n",
    "        # f1 = f1_score(label_flat, output_flat)\n",
    "\n",
    "        rst_list.append((precision, recall, accuracy, f1, iou))\n",
    "    \n",
    "    # avg_precision = sum(item[0] for item in rst_list) / len(rst_list)\n",
    "    # avg_recall = sum(item[1] for item in rst_list) / len(rst_list)\n",
    "    # avg_accuracy = sum(item[2] for item in rst_list) / len(rst_list)\n",
    "    # avg_f1 = sum(item[3] for item in rst_list) / len(rst_list)\n",
    "    # avg_iou = sum(item[4] for item in rst_list) / len(rst_list)\n",
    "    avg_precision = np.mean([item[0] for item in rst_list])\n",
    "    avg_recall = np.mean([item[1] for item in rst_list])\n",
    "    avg_accuracy = np.mean([item[2] for item in rst_list])\n",
    "    avg_f1 = np.mean([item[3] for item in rst_list])\n",
    "    avg_iou = np.mean([item[4] for item in rst_list])\n",
    "    print(f'{os.path.basename(base_dir)} - precision: {avg_precision:.3f}, recall: {avg_recall:.3f}, accuracy: {avg_accuracy:.3f}, f1: {avg_f1:.3f}, iou: {avg_iou:.3f}')\n",
    "\n",
    "    total_tp = np.sum(tp_list)\n",
    "    total_tn = np.sum(tn_list)\n",
    "    total_fp = np.sum(fp_list)\n",
    "    total_fn = np.sum(fn_list)\n",
    "    y_true = np.concatenate([np.ones(total_tp + total_fn), np.zeros(total_tn + total_fp)])\n",
    "    y_score = np.concatenate([np.ones(total_tp), np.zeros(total_fn), np.ones(total_fp), np.zeros(total_tn)])\n",
    "\n",
    "    lst_cdata.append((y_true, y_score))\n",
    "    lst_rst.append((losses[idx], avg_precision, avg_recall, avg_accuracy, avg_f1, avg_iou))\n",
    "    idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_loss_list = [item[0] for item in lst_rst]\n",
    "avg_precision_list = [item[1] for item in lst_rst]\n",
    "avg_recall_list = [item[2] for item in lst_rst]\n",
    "avg_accuracy_list = [item[3] for item in lst_rst]\n",
    "avg_f1_list = [item[4] for item in lst_rst]\n",
    "avg_iou_list = [item[5] for item in lst_rst]\n",
    "model_list = ['BCE', 'Dice', 'Focal', 'L1', 'L2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58372 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model_list, avg_loss_list, '-', color='blue', label=f'Avg Loss')\n",
    "plt.plot(model_list, avg_precision_list, '-', color='green', label=f'Avg Precision')\n",
    "plt.plot(model_list, avg_recall_list, '-', color='red', label=f'Avg Recall')\n",
    "plt.plot(model_list, avg_accuracy_list, '-', color='gold', label=f'Avg Accuracy')\n",
    "plt.plot(model_list, avg_f1_list, '-', color='orange', label=f'Avg F1-Score')\n",
    "plt.plot(model_list, avg_iou_list, '-', color='pink', label=f'mIoU')\n",
    "plt.title('Performance metrics for each test case')\n",
    "plt.legend(loc='right', bbox_to_anchor=(0.99, 0.3))\n",
    "# plt.legend(loc='lower right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unet PR AUC: 0.704, minDist: 0.497, (0.885424512257604, 0.51682025846915)\n",
      "unet-dice-loss PR AUC: 0.692, minDist: 0.555, (0.931617865125619, 0.44876511258753)\n",
      "unet-focal-loss PR AUC: 0.822, minDist: 0.277, (0.7487617009547043, 0.8840717896362793)\n",
      "unet-l1 PR AUC: 0.754, minDist: 0.393, (0.8766366499467493, 0.626387811957221)\n",
      "unet-l2 PR AUC: 0.760, minDist: 0.349, (0.7813983872652583, 0.7285129156027101)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 79876 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve, auc\n",
    "\n",
    "idx = 0\n",
    "for cdata in lst_cdata:\n",
    "    y_true, y_score = cdata\n",
    "\n",
    "    # PR 커브 계산 및 그리기\n",
    "    precision, recall, _ = precision_recall_curve(y_true, y_score)\n",
    "\n",
    "    # AUC 계산\n",
    "    pr_auc = auc(recall, precision)\n",
    "    plt.plot(recall, precision, '-', color=colors[idx], label=f'{os.path.basename(base_dirs[idx])}(AUC = {pr_auc:.3f})')\n",
    "\n",
    "    # (1, 1)에 가장 가까운 점 찾기\n",
    "    min_distance = float('inf')\n",
    "    closest_point = None\n",
    "    for i in range(len(recall)):\n",
    "        distance = ((1 - recall[i])**2 + (1 - precision[i])**2)**0.5\n",
    "        if distance < min_distance:\n",
    "            min_distance = distance\n",
    "            closest_point = i\n",
    "    plt.scatter(recall[closest_point], precision[closest_point], color=colors[idx], marker='o')\n",
    "\n",
    "    print(f'{os.path.basename(base_dirs[idx])} PR AUC: {pr_auc:.3f}, minDist: {min_distance:.3f}, {(recall[closest_point], precision[closest_point])}')\n",
    "    idx += 1\n",
    "\n",
    "plt.plot([0.0, 1.05], [0.0, 1.05], '--', color='navy', label='baseline')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.title('PR Curve')\n",
    "plt.legend()\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unet ROC AUC: 0.760, minDist: 0.121, (0.03855843283619862, 0.885424512257604)\n",
      "unet-dice-loss ROC AUC: 0.760, minDist: 0.087, (0.05330329573045489, 0.931617865125619)\n",
      "unet-focal-loss ROC AUC: 0.760, minDist: 0.251, (0.004573449270014527, 0.7487617009547043)\n",
      "unet-l1 ROC AUC: 0.760, minDist: 0.126, (0.024355441418179306, 0.8766366499467493)\n",
      "unet-l2 ROC AUC: 0.760, minDist: 0.219, (0.013563852969138058, 0.7813983872652583)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 62832 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "\n",
    "idx = 0\n",
    "for cdata in lst_cdata:\n",
    "    y_true, y_score = cdata\n",
    "\n",
    "    # AUC 계산\n",
    "    auc = roc_auc_score(y_true, y_score)\n",
    "\n",
    "    # ROC 커브 계산 및 그리기\n",
    "    fpr, tpr, _ = roc_curve(y_true, y_score)\n",
    "    plt.plot(fpr, tpr, color=colors[idx], label=f'{os.path.basename(base_dirs[idx])}(AUC = {auc:.3f})')\n",
    "\n",
    "    # (0, 1)에 가장 가까운 점 찾기\n",
    "    min_distance = float('inf')\n",
    "    closest_point = None\n",
    "    for i in range(len(fpr)):\n",
    "        distance = ((0 - fpr[i])**2 + (1 - tpr[i])**2)**0.5\n",
    "        if distance < min_distance:\n",
    "            min_distance = distance\n",
    "            closest_point = i\n",
    "    plt.scatter(fpr[closest_point], tpr[closest_point], color=colors[idx], marker='o')\n",
    "\n",
    "    print(f'{os.path.basename(base_dirs[idx])} ROC AUC: {pr_auc:.3f}, minDist: {min_distance:.3f}, {(fpr[closest_point], tpr[closest_point])}')\n",
    "    idx += 1\n",
    "\n",
    "plt.plot([0.0, 1.05], [0.0, 1.05], '--', color='navy', label='baseline')\n",
    "plt.xlabel('FPR')\n",
    "plt.ylabel('TPR')\n",
    "plt.title('ROC Curve')\n",
    "plt.legend()\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 228560 characters]",
      "text/plain": [
       "<Figure size 800x950 with 56 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# id = 7\n",
    "h, w = 0, 2\n",
    "lst_idx = [857,739,986,2,592,333,487,155]\n",
    "names = ['BCE', 'Dice', 'Focal', 'L1', 'L2']\n",
    "fig, axs = plt.subplots(len(lst_idx), 7, figsize=(8, 9.5))\n",
    "\n",
    "for id in lst_idx:\n",
    "    result_dir = os.path.join(base_dirs[0], 'result')\n",
    "    img = np.load(os.path.join(result_dir,\"numpy\", lst_img[id]))\n",
    "    gt = np.load(os.path.join(result_dir,\"numpy\", lst_gt[id]))\n",
    "    axs[h,0].imshow(img, cmap='gray')\n",
    "    axs[h,0].axis('off')\n",
    "    if h == 0:\n",
    "        axs[h,0].set_title(f'Image')\n",
    "    axs[h,1].imshow(gt, cmap='gray')\n",
    "    axs[h,1].axis('off')\n",
    "    if h == 0:\n",
    "        axs[h,1].set_title(f'GT')\n",
    "\n",
    "    for base_dir in base_dirs:\n",
    "        result_dir = os.path.join(base_dir, 'result')\n",
    "\n",
    "        ##\n",
    "        lst_data = os.listdir(os.path.join(result_dir, 'numpy'))\n",
    "\n",
    "        lst_gt = [f for f in lst_data if f.startswith('gt')]\n",
    "        lst_pr = [f for f in lst_data if f.startswith('pr')]\n",
    "\n",
    "        lst_gt.sort()\n",
    "        lst_pr.sort()\n",
    "\n",
    "        ##\n",
    "        pr = np.load(os.path.join(result_dir,\"numpy\", lst_pr[id]))\n",
    "        axs[h,w].imshow(pr, cmap='gray')\n",
    "        axs[h,w].axis('off')\n",
    "        if h == 0:\n",
    "            axs[h,w].set_title(f'{names[w-2]}')\n",
    "        w += 1\n",
    "        if w == 7:\n",
    "            w = 2\n",
    "            h += 1\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
