feat: add VinBigData CXR Kaggle pipeline notebooks
This commit is contained in:
@@ -0,0 +1,303 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"papermill": {
|
||||
"duration": 0.005268,
|
||||
"end_time": "2021-03-21T01:43:26.398126",
|
||||
"exception": false,
|
||||
"start_time": "2021-03-21T01:43:26.392858",
|
||||
"status": "completed"
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"### Credit of this notebook goes entirely to below public notebook, kindly upvote and appreciate the original author\n",
|
||||
"\n",
|
||||
"* https://www.kaggle.com/muhammad4hmed/lets-overfit-together"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
|
||||
"_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
|
||||
"execution": {
|
||||
"iopub.execute_input": "2021-03-21T01:43:26.411073Z",
|
||||
"iopub.status.busy": "2021-03-21T01:43:26.410014Z",
|
||||
"iopub.status.idle": "2021-03-21T01:43:26.414755Z",
|
||||
"shell.execute_reply": "2021-03-21T01:43:26.415196Z"
|
||||
},
|
||||
"papermill": {
|
||||
"duration": 0.012965,
|
||||
"end_time": "2021-03-21T01:43:26.415505",
|
||||
"exception": false,
|
||||
"start_time": "2021-03-21T01:43:26.402540",
|
||||
"status": "completed"
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"import numpy as np # linear algebra\n",
|
||||
"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
|
||||
"\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"execution": {
|
||||
"iopub.execute_input": "2021-03-21T01:43:26.427333Z",
|
||||
"iopub.status.busy": "2021-03-21T01:43:26.426414Z",
|
||||
"iopub.status.idle": "2021-03-21T01:43:26.475008Z",
|
||||
"shell.execute_reply": "2021-03-21T01:43:26.475504Z"
|
||||
},
|
||||
"papermill": {
|
||||
"duration": 0.055753,
|
||||
"end_time": "2021-03-21T01:43:26.475682",
|
||||
"exception": false,
|
||||
"start_time": "2021-03-21T01:43:26.419929",
|
||||
"status": "completed"
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>image_id</th>\n",
|
||||
" <th>target</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>002a34c58c5b758217ed1f584ccbcfe9</td>\n",
|
||||
" <td>0.013326</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>004f33259ee4aef671c2b95d54e4be68</td>\n",
|
||||
" <td>0.037235</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>008bdde2af2462e86fd373a445d0f4cd</td>\n",
|
||||
" <td>0.939700</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>009bc039326338823ca3aa84381f17f1</td>\n",
|
||||
" <td>0.123799</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>00a2145de1886cb9eb88869c85d74080</td>\n",
|
||||
" <td>0.654006</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>...</th>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2995</th>\n",
|
||||
" <td>ff91fb82429a27521bbec8569b041f02</td>\n",
|
||||
" <td>0.936325</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2996</th>\n",
|
||||
" <td>ff9fcc4087ed5e941209aa3fa948e364</td>\n",
|
||||
" <td>0.963583</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2997</th>\n",
|
||||
" <td>ffaa288c8abca300974f043b57d81521</td>\n",
|
||||
" <td>0.178720</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2998</th>\n",
|
||||
" <td>ffc441e0c8b7153844047483a577e7c3</td>\n",
|
||||
" <td>0.225196</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2999</th>\n",
|
||||
" <td>ffccf1709d0081d122a1d1f9edbefdf1</td>\n",
|
||||
" <td>0.987406</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"<p>3000 rows × 2 columns</p>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" image_id target\n",
|
||||
"0 002a34c58c5b758217ed1f584ccbcfe9 0.013326\n",
|
||||
"1 004f33259ee4aef671c2b95d54e4be68 0.037235\n",
|
||||
"2 008bdde2af2462e86fd373a445d0f4cd 0.939700\n",
|
||||
"3 009bc039326338823ca3aa84381f17f1 0.123799\n",
|
||||
"4 00a2145de1886cb9eb88869c85d74080 0.654006\n",
|
||||
"... ... ...\n",
|
||||
"2995 ff91fb82429a27521bbec8569b041f02 0.936325\n",
|
||||
"2996 ff9fcc4087ed5e941209aa3fa948e364 0.963583\n",
|
||||
"2997 ffaa288c8abca300974f043b57d81521 0.178720\n",
|
||||
"2998 ffc441e0c8b7153844047483a577e7c3 0.225196\n",
|
||||
"2999 ffccf1709d0081d122a1d1f9edbefdf1 0.987406\n",
|
||||
"\n",
|
||||
"[3000 rows x 2 columns]"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pred_2class = pd.read_csv(\"../input/vinbigdata-2class-prediction/2-cls test pred.csv\")\n",
|
||||
"low_threshold = 0.0\n",
|
||||
"high_threshold = 0.90\n",
|
||||
"pred_2class"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"papermill": {
|
||||
"duration": 0.004459,
|
||||
"end_time": "2021-03-21T01:43:26.485337",
|
||||
"exception": false,
|
||||
"start_time": "2021-03-21T01:43:26.480878",
|
||||
"status": "completed"
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"## Apply 2class filter"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"execution": {
|
||||
"iopub.execute_input": "2021-03-21T01:43:26.504745Z",
|
||||
"iopub.status.busy": "2021-03-21T01:43:26.503972Z",
|
||||
"iopub.status.idle": "2021-03-21T01:43:27.617091Z",
|
||||
"shell.execute_reply": "2021-03-21T01:43:27.616495Z"
|
||||
},
|
||||
"papermill": {
|
||||
"duration": 1.127304,
|
||||
"end_time": "2021-03-21T01:43:27.617254",
|
||||
"exception": false,
|
||||
"start_time": "2021-03-21T01:43:26.489950",
|
||||
"status": "completed"
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"n_normal: 1586 -> 1974 with threshold 0.0 & 0.9\n",
|
||||
"Keep 0 Add 1026 Replace 1974\n",
|
||||
"Saved to submission.csv\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"NORMAL = \"14 1 0 0 1 1\"\n",
|
||||
"\n",
|
||||
"pred_det_df = pd.read_csv(\"../input/vinbigdatastack/submission_postprocessed.csv\")\n",
|
||||
"n_normal_before = len(pred_det_df.query(\"PredictionString == @NORMAL\"))\n",
|
||||
"merged_df = pd.merge(pred_det_df, pred_2class, on=\"image_id\", how=\"left\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if \"target\" in merged_df.columns:\n",
|
||||
" merged_df[\"class0\"] = 1 - merged_df[\"target\"]\n",
|
||||
"\n",
|
||||
"c0, c1, c2 = 0, 0, 0\n",
|
||||
"for i in range(len(merged_df)):\n",
|
||||
" p0 = merged_df.loc[i, \"class0\"]\n",
|
||||
" if p0 < low_threshold:\n",
|
||||
"\n",
|
||||
" c0 += 1\n",
|
||||
" elif low_threshold <= p0 and p0 < high_threshold:\n",
|
||||
"\n",
|
||||
" merged_df.loc[i, \"PredictionString\"] += f\" 14 {p0} 0 0 1 1\"\n",
|
||||
" c1 += 1\n",
|
||||
" else:\n",
|
||||
"\n",
|
||||
" merged_df.loc[i, \"PredictionString\"] = NORMAL\n",
|
||||
" c2 += 1\n",
|
||||
"\n",
|
||||
"n_normal_after = len(merged_df.query(\"PredictionString == @NORMAL\"))\n",
|
||||
"print(\n",
|
||||
" f\"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}\"\n",
|
||||
")\n",
|
||||
"print(f\"Keep {c0} Add {c1} Replace {c2}\")\n",
|
||||
"submission_filepath = str(\"submission.csv\")\n",
|
||||
"submission_df = merged_df[[\"image_id\", \"PredictionString\"]]\n",
|
||||
"submission_df.to_csv(submission_filepath, index=False)\n",
|
||||
"print(f\"Saved to {submission_filepath}\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.9"
|
||||
},
|
||||
"papermill": {
|
||||
"default_parameters": {},
|
||||
"duration": 7.332473,
|
||||
"end_time": "2021-03-21T01:43:28.131726",
|
||||
"environment_variables": {},
|
||||
"exception": null,
|
||||
"input_path": "__notebook__.ipynb",
|
||||
"output_path": "__notebook__.ipynb",
|
||||
"parameters": {},
|
||||
"start_time": "2021-03-21T01:43:20.799253",
|
||||
"version": "2.2.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
Reference in New Issue
Block a user