remove unnecessary dependencies

Author: ChristofHenkel

competitions/kaggle/Cryo-ET/1st_place_solution/README.md

@@ -66,7 +66,7 @@ We provide three different configurations which differ only in the used backbone
 
 ```python train.py -C cfg_resnet34 --output_dir WHATEVERISYOUROUTPUTDIR```
 
-This will save checkpoints under the specified WHATEVERISYOUROUTPUTDIR.
+This will save checkpoints under the specified WHATEVERISYOUROUTPUTDIR when training is finished.
 By default models are trained using bfloat16 which requires a GPU capable of that. Alternatively you can set ```cfg.bf16=False``` or overwrite as flag ```--bf16 False``` when running ```train.py ```.
 
 ### Replicating 1st place solution (segmentation part)

competitions/kaggle/Cryo-ET/1st_place_solution/metrics/metric_1.py

@@ -1,15 +1,7 @@
 import numpy as np
 import torch
 from sklearn.metrics import roc_auc_score
-
-"""
-Derived from:
-https://github.com/cellcanvas/album-catalog/blob/main/solutions/copick/compare-picks/solution.py
-"""
-
-import numpy as np
 import pandas as pd
-
 from scipy.spatial import KDTree
 
 
@@ -156,13 +148,6 @@ def calc_metric(cfg, pp_out, val_df, pre="val"):
     submission['experiment'] = solution['experiment'].unique()[0]
     submission['id'] = range(len(submission))
 
-#     score003 = score(
-#         solution.copy(),
-#         submission[submission['conf']>0.03].copy(),
-#         row_id_column_name = 'id',
-#         distance_multiplier=0.5,
-#         beta=4)[0]
-#     print('score003',score003)
 
     best_ths = []
     for p in particles:
@@ -196,42 +181,5 @@ def calc_metric(cfg, pp_out, val_df, pre="val"):
     result['score'] = score_pp
 #     print(result)
     return result
-# #     if isinstance(pred_df,list):
-# #         pred_df,gt_df = pred_df
-# #     else:
-# #         gt_df = None
-
-#     y_true = val_df['score'].values
-#     y_pred = val_data['preds'].cpu().numpy()
-#     score = get_score(y_true.flatten(), y_pred.flatten())
-# #     print(score)
-
-# #     df['score'] = df['location'].apply(ast.literal_eval)
-# #     df['span'] = df['location'].apply(location_to_span)
-# #     spans_true = df['span'].values
-
-# #     df_pred = pred_df.copy()
-# #     # df_pred['location'] = df_pred['location'].apply(ast.literal_eval)
-# #     df_pred['span'] = df_pred['pred_location'].apply(pred_location_to_span)
-# #     spans_pred = df_pred['span'].values
-
-# #     score = span_micro_f1(spans_pred, spans_true)
-
-#     if hasattr(cfg, "neptune_run"):
-#         cfg.neptune_run[f"{pre}/score/"].log(score, step=cfg.curr_step)
-#         print(f"{pre} score: {score:.6}")
-# #     else:
-# #         return score
-
-# #     if gt_df is not None:
-# #         df_pred = gt_df.copy()
-# #         df_pred['span'] = df_pred['pred_location'].apply(pred_location_to_span)
-# #         spans_pred = df_pred['span'].values
-
-# #         score = span_micro_f1(spans_pred, spans_true)
-
-# #         if hasattr(cfg, "neptune_run"):
-# #             cfg.neptune_run[f"{pre}/score_debug/"].log(score, step=cfg.curr_step)
-# # #             print(f"{pre} score_debug: {score:.6}")          
-#     return score
+
 

competitions/kaggle/Cryo-ET/1st_place_solution/postprocess/pp_1.py

@@ -1,5 +1,3 @@
-
-
 import pandas as pd
 import torch
 from torch.nn import functional as F
@@ -8,8 +6,6 @@
 import torch
 from torch import nn
 
-
-
 def simple_nms(scores, nms_radius: int):
     """ Fast Non-maximum suppression to remove nearby points """
     assert(nms_radius >= 0)
@@ -54,7 +50,6 @@ def post_process_pipeline(cfg, val_data, val_df):
         pred_df_ = pd.DataFrame(xyz.cpu().numpy(),columns=['x','y','z'])
         pred_df_['particle_type'] = p
         pred_df_['conf'] = conf.cpu().numpy()
-#         pred_df_['experiment'] = experiments[fold]
         pred_df += [pred_df_]
     pred_df = pd.concat(pred_df)
     pred_df = pred_df[(pred_df['x']<6300) & (pred_df['y']<6300)& (pred_df['z']<1840) & (pred_df['conf']>0.01)].copy()

competitions/kaggle/Cryo-ET/1st_place_solution/requirements.txt

@@ -1,10 +1,2 @@
-kaggle
-optuna
-boto3
-neptune
 zarr
-albumentations==1.4.21
-opencv-python==4.5.5.64
-timm==1.0.11
 monai==1.4.0
-mrcfile

competitions/kaggle/Cryo-ET/1st_place_solution/train.py

@@ -29,8 +29,6 @@
 from utils import (
     get_optimizer,
     get_scheduler,
-    setup_neptune,
-    upload_s3,
 )
 
 
@@ -120,9 +118,7 @@ def run_eval(model, val_dataloader, cfg, pre="val", curr_epoch=0):
             loss = np.mean(losses)
 
             print(f"Mean {pre}_{k}", loss)
-            if cfg.neptune_run:
-                if not math.isinf(loss) and not math.isnan(loss):
-                    cfg.neptune_run[f"{pre}/{k}"].log(loss, step=cfg.curr_step)
+
 
 
     if (cfg.local_rank == 0) and (cfg.calc_metric) and (((curr_epoch + 1) % cfg.calc_metric_epochs) == 0):
@@ -135,9 +131,7 @@ def run_eval(model, val_dataloader, cfg, pre="val", curr_epoch=0):
 
         for k, v in val_score.items():
             print(f"{pre}_{k}: {v:.3f}")
-            if cfg.neptune_run:
-                if not math.isinf(v) and not math.isnan(v):
-                    cfg.neptune_run[f"{pre}/{k}"].log(v, step=cfg.curr_step)
+
 
     if cfg.distributed:
         torch.distributed.barrier()
@@ -191,8 +185,7 @@ def train(cfg):
 
     set_seed(cfg.seed)
 
-    if cfg.local_rank == 0:
-        cfg.neptune_run = setup_neptune(cfg)
+
 
     train_df, val_df, test_df = get_data(cfg)
 
@@ -347,17 +340,7 @@ def train(cfg):
                 if cfg.local_rank == 0 and cfg.curr_step % cfg.batch_size == 0:
 
                     loss_names = [key for key in output_dict if 'loss' in key]
-                    if cfg.neptune_run:
-                        for l in loss_names:
-                            v = output_dict[l].item()
-                            if not math.isinf(v) and not math.isnan(v):
-                                cfg.neptune_run[f"train/{l}"].log(value=v, step=cfg.curr_step)
-                        cfg.neptune_run["lr"].log(value=optimizer.param_groups[0]["lr"], step=cfg.curr_step)
-                        if total_grad_norm is not None:
-                            cfg.neptune_run["total_grad_norm"].log(value=total_grad_norm.item(), step=cfg.curr_step)
-                            cfg.neptune_run["total_grad_norm_after_clip"].log(value=total_grad_norm_after_clip.item(), step=cfg.curr_step)
-                        if total_weight_norm is not None:
-                            cfg.neptune_run["total_weight_norm"].log(value=total_weight_norm.item(), step=cfg.curr_step)
+
                     progress_bar.set_description(f"loss: {np.mean(losses[-10:]):.4f}")
 
                 if cfg.eval_steps != 0:
@@ -426,14 +409,11 @@ def train(cfg):
     parser = argparse.ArgumentParser(description="")
 
     parser.add_argument("-C", "--config", help="config filename")
-    parser.add_argument("-D", "--debug", action='store_true', help="debugging True/ False")
     parser_args, other_args = parser.parse_known_args(sys.argv)
 
     cfg = copy(importlib.import_module(parser_args.config).cfg)
 
-    if parser_args.debug:
-        print('debug mode')
-        cfg.neptune_connection_mode = 'debug'
+
 
 
     # overwrite params in config with additional args

competitions/kaggle/Cryo-ET/1st_place_solution/utils.py

@@ -1,18 +1,14 @@
-
 import random
 import os
 import numpy as np
 import pandas as pd
 import torch
 from torch.utils.data import Sampler, RandomSampler, SequentialSampler, DataLoader, WeightedRandomSampler
 from torch import nn, optim
-# from torch.optim import AdamW
 from torch.optim.lr_scheduler import LambdaLR
 from torch.optim import lr_scheduler
 import importlib
 import math
-import neptune
-from neptune.utils import stringify_unsupported
 
 import logging
 import pickle
@@ -469,22 +465,7 @@ def get_scheduler(cfg, optimizer, total_steps):
     return scheduler
 
 
-def setup_neptune(cfg):
-    neptune_run = None
-    if cfg.neptune_project:
-        neptune_run = neptune.init_run(
-            project=cfg.neptune_project,
-#             tags=cfg.tags,
-            mode=cfg.neptune_connection_mode,
-            capture_stdout=False,
-            capture_stderr=False,
-            source_files=[f'models/{cfg.model}.py',f'data/{cfg.dataset}.py',f'configs/{cfg.name}.py']
-        )
-
-
-        neptune_run["cfg"] = stringify_unsupported(cfg.__dict__)
 
-    return neptune_run
 
 
 def read_df(fn):