vinbigdata-cxr-kaggle-pipeline/ensemble-of-best-public-notebooks.ipynb

Credit of this notebook goes entirely to below public notebook, kindly upvote and appreciate the original author

• https://www.kaggle.com/muhammad4hmed/lets-overfit-together

In [1]:

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

In [2]:

pred_2class = pd.read_csv("../input/vinbigdata-2class-prediction/2-cls test pred.csv")
low_threshold = 0.0
high_threshold = 0.90
pred_2class

Out [2]:

	image_id	target
0	002a34c58c5b758217ed1f584ccbcfe9	0.013326
1	004f33259ee4aef671c2b95d54e4be68	0.037235
2	008bdde2af2462e86fd373a445d0f4cd	0.939700
3	009bc039326338823ca3aa84381f17f1	0.123799
4	00a2145de1886cb9eb88869c85d74080	0.654006
...	...	...
2995	ff91fb82429a27521bbec8569b041f02	0.936325
2996	ff9fcc4087ed5e941209aa3fa948e364	0.963583
2997	ffaa288c8abca300974f043b57d81521	0.178720
2998	ffc441e0c8b7153844047483a577e7c3	0.225196
2999	ffccf1709d0081d122a1d1f9edbefdf1	0.987406

3000 rows × 2 columns

Apply 2class filter

In [3]:

NORMAL = "14 1 0 0 1 1"

pred_det_df = pd.read_csv("../input/vinbigdatastack/submission_postprocessed.csv")
n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")


if "target" in merged_df.columns:
    merged_df["class0"] = 1 - merged_df["target"]

c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)):
    p0 = merged_df.loc[i, "class0"]
    if p0 < low_threshold:

        c0 += 1
    elif low_threshold <= p0 and p0 < high_threshold:

        merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
        c1 += 1
    else:

        merged_df.loc[i, "PredictionString"] = NORMAL
        c2 += 1

n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
    f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}"
)
print(f"Keep {c0} Add {c1} Replace {c2}")
submission_filepath = str("submission.csv")
submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
print(f"Saved to {submission_filepath}")

n_normal: 1586 -> 1974 with threshold 0.0 & 0.9
Keep 0 Add 1026 Replace 1974
Saved to submission.csv