\section{Algorithm of the proposed method}
\label{sec:algorithm}
    \begin{algorithm}[ht]
        \caption{Training procedure of the proposed INOLML.
        }
        \label{algorithm:training_procedure}
        \begin{algorithmic}[1]
            \Procedure{Training}{$\mathcal{D}$, $\eta$, $T$, $\widetilde{T}$, $T^{(u)}$, $\widetilde{\eta}$, $\kappa,N, M, K, C$}
                \LComment{\(\mathcal{D}\): noisy dataset}
                \LComment{\(\eta\): learning rate per iteration}
                \LComment{\(T\): total number of iterations}
                \LComment{\(\widetilde{T}\): minimum number of iterations before  updating the moving average robust labels}
                \LComment{\(T^{(u)}\): interval between updates}
                \LComment{\(\widetilde{\eta}\): learning rate threshold}
                \LComment{\(\kappa,N, M, K, C\): hyper-parameters}

                \Statex

               
                \State Warmup model $f_{\theta}(.)$ with \(\ell_{\mathrm{CE}}(.)\) from $\mathcal{D}$
                \State $\mathcal{D}^{(c)}=\mathsf{PseudoCleanDetector}\left (\mathcal{D} \right )$ using Equation \ref{eq:PseudoCleanDetector}
                \State Initialise moving average robust label $\{\tilde{\mathbf{y}}_i\}_{i=1}^{|\mathcal{D}^{(c)}|}$ of samples in $\mathcal{D}^{(c)}$
                \State Initialise $\mathcal{D}^{(v)}$ and $\mathcal{D}^{(t)}$ from $\mathcal{D}^{(c)}$  using 
               
                 Equation \ref{eq:select_validation_set}
               
                \State Reinitialize model $f_{\theta}(.)$
                \For{$t = 1$ to $T$}
                    \State Meta-learn to train  \(\theta\), \(\omega\) and \(\lambda\) using Equation \ref{eq:meta_learning_bilevel_optimisation} 
                   
                  
                    \State Update  $\{\tilde{\mathbf{y}}_i\}_{i=1}^{|\mathcal{D}^{(c)}|}$ of samples in $\mathcal{D}^{(c)}$ if  $ t > \widetilde{T}$ and  $\eta_t < \widetilde{\eta}$

                    \State Update $\mathcal{D}^{(c)}=\mathsf{PseudoCleanDetector}\left (\mathcal{D} \right )$ using Equation \ref{eq:PseudoCleanDetector} if  $\eta_t = 0$
                    \If{$T^{(u)} \equiv t \mod T^{(u)}$}
                        \State Update $\mathcal{D}^{(v)}$ and $\mathcal{D}^{(t)}$ from $\mathcal{D}^{(c)}$  using 
               
                 Equation \ref{eq:select_validation_set}
                       
                       
                       
                       
                       
                    \EndIf
                \EndFor
                \State \Return the trained model parameter \(\theta\)
            \EndProcedure
        \end{algorithmic}
    \end{algorithm}
    



\section{Implementation Details}
\label{sec:implementation_details_appendix}
All CIFAR experiments use batches of size 100, which are trained on a single GPU. Similar to the Distill noise model~\cite{Distill_noise}, we use \(p=5, k=20\) for CIFAR experiments, except the ones with the imbalance setting.

For Red Mini-ImageNet experiments, we trained the model on a single  GPU with batches of size 100, with \(p=5, k=10\).

For the WebVision experiment, we use \(p=4, k=8\) with 4 NVIDIA V100 GPU and batches of size 16.  All experiments use \(N=200, K=50, \kappa = 0.9\).

\newpage
\section{Additional Results of Symmetric Noise on CIFAR Datasets}
\label{sec:additional_symmetric_noise_results}
        We provide additional symmetric noise results of our proposed method and the Distill model~\cite{Distill_noise} in Table \ref{tab:table_1}. Note that our method is markedly better than Distill, particularly for the simpler model (RN29) with few samples per class (1 and 5) in the validation set. For the more complex model (WRN) and large validation set (10 samples per class), our method is still better than Distill, except for CIFAR100 at 0.8 symmetric noise rate.

        \begin{table*}[h]
        \centering
        \caption{Test accuracy (in \%) comparison between  our method ('INOLML') and the Distill noise ('DN') on symmetric noise using 1, 5 and 10 samples per class in the validation set on two backbone models: Resnet29 ('RN29') and Wideresnet28-10 ('WRN'). The results of the Distill model with WideResnet28-10 are collected from \cite{Distill_noise}.
        Recall that the Distill needs a clean set, while INOLML works with a pseudo-clean set.}
        
        \label{tab:table_1}
        \scalebox{1.0}{
        \begin{tabular}{l c l | l | l | l | l | l}
            \toprule
            \multirow{3}{*}{\bfseries Method} & \multirow{3}{*}{\shortstack{\bfseries Val. Set\\ \bfseries size}} & \multicolumn{6}{c}{\bfseries Dataset}  \\
           
            & & \multicolumn{3}{c}{\bfseries CIFAR10} & \multicolumn{3}{c}{\bfseries CIFAR100}  \\
            \cmidrule{3-8} 
            & & \multicolumn{1}{c}{\bfseries 0.2} & \multicolumn{1}{c}{\bfseries 0.4} & \multicolumn{1}{c|}{\bfseries 0.8} & \multicolumn{1}{c}{\bfseries 0.2} & \multicolumn{1}{c}{\bfseries 0.4} & \multicolumn{1}{c}{\bfseries 0.8} \\
            \midrule
            DN-RN29  & \multirow{2}{*}{1} & 87.0 \(\pm\) 0.5 & 85.3 \(\pm\) 0.5 & \multicolumn{1}{c|}{FAIL} & 58.9 \(\pm\)  0.5 & 55.8 \(\pm\) 0.7 & \multicolumn{1}{c}{FAIL} \\
            INOLML-RN29  & & 90.3 \(\pm\) 0.2 & 89.1 \(\pm\) 0.5 & 79.1 \(\pm\) 0.3 & 65.9 \(\pm\) 0.2 & 61.5 \(\pm\) 0.2 & 55.1 \(\pm\) 0.1 \\
            \midrule
            DN-RN29  & \multirow{2}{*}{5} & 90.7 \(\pm\) 0.3 &89.0 \(\pm\) 0.3 & 83.5 \(\pm\) 0.2 & 62.6 \(\pm\)  0.4 & 58.8 \(\pm\) 0.5 & 48.5 \(\pm\) 0.5 \\
            INOLML-RN29  & & 90.9 \(\pm\) 0.2 & 90.9 \(\pm\) 0.1&87.4 \(\pm\)  0.2&66.6 \(\pm\)  0.1&65.7 \(\pm\)  0.1&59.0 \(\pm\) 0.5 \\
            \midrule
            DN-RN29  & \multirow{2}{*}{10} & 91.0 \(\pm\) 0.2 & 89.2 \(\pm\) 0.1&87.0 \(\pm\) 0.1&63.7 \(\pm\)  0.2&60.5 \(\pm\) 0.2&57.5  \(\pm\) 0.5 \\
            INOLML-RN29  & & 92.2 \(\pm\) 0.1 & 91.0 \(\pm\) 0.1 & 87.9 \(\pm\)  0.2 & 67.1 \(\pm\) 0.1 & 66.3 \(\pm\)  0.1 & 59.2 \(\pm\) 0.2 \\
            \midrule
            \midrule
            DN-WRN  & \multirow{2}{*}{1} & 95.4 \(\pm\) 0.6 & 94.5 \(\pm\) 1.0 & 87.9 \(\pm\) 5.1 & 77.4 \(\pm\) 0.4 & 75.5 \(\pm\) 1.1 & 62.1 \(\pm\) 1.2 \\
            INOLML-WRN  & & 96.0 \(\pm\) 0.2 &95.9 \(\pm\) 0.2&94.3 \(\pm\) 0.2&81.6 \(\pm\) 0.2&79.5 \(\pm\) 0.2&73.6 \(\pm\) 0.3 \\
            \midrule
            DN-WRN  & \multirow{2}{*}{5} & 96.4 \(\pm\) 0.0 & 95.5 \(\pm\) 0.6 & 91.8  \(\pm\)  3.0 & 80.4 \(\pm\) 0.5 & 79.6  \(\pm\)  0.3 & 73.6 \(\pm\) 1.5 \\
            INOLML-WRN & & 96.4 \(\pm\) 0.1 &96.2 \(\pm\) 0.1 & 94.6 \(\pm\) 0.2 & 82.2 \(\pm\) 0.2&81.5 \(\pm\) 0.2&74.5 \(\pm\)  0.2 \\
            \midrule
            DN-WRN & \multirow{2}{*}{10} & 96.2 \(\pm\) 0.2 &95.9 \(\pm\) 0.2&93.7  \(\pm\)  0.5&81.2 \(\pm\) 0.7&80.2  \(\pm\)  0.3 & \(\mathbf{75.5 \pm 0.2}\) \\
            INOLML-WRN & & \(\mathbf{96.9 \pm 0.1}\) & \(\mathbf{96.6 \pm 0.1}\) & \(\mathbf{95.0 \pm 0.2}\) & \(\mathbf{82.0 \pm 0.2}\) & \(\mathbf{81.3 \pm 0.2}\) & 74.7 \(\pm\) 0.1 \\
            \bottomrule
        \end{tabular}
        }
    \end{table*}

   
   
        
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   

\newpage

        



\section{Ablation Study and Discussion}
\vspace{-0.5em}
   
    
    
        We first study the optimisation in~\cref{eq:select_validation_set}.
        In the lower-lever optimisation of \cref{eq:select_validation_set}, the function $\mathsf{Info(.)}$ not just select samples that  maximise the training sample weight from~\cref{eq:gradient_w},  as that may lead to scenarios where most of selected samples are located in the same region of the feature space. Instead, we also maximise a diversity factor defined by maximising the maximum \say{information content} that each training sample can get from any  sample in the clean validation set.  In~\cref{tab:ablatant}, we show an ablation study about the importance of this factor by redefining $\mathsf{Info(.)}$ in \cref{eq:select_validation_set} with \cref{eq:gradient_w} (\textbf{Weight in \cref{eq:gradient_w}}). We also study the role of
        $\mathsf{Clean}(.)$ 
        in~\cref{eq:select_validation_set} by optimising only the lower part of \cref{eq:select_validation_set} (\textbf{$\mathsf{Info}(.)$ Only}). This ablation study is conducted on CIFAR10 and CIFAR100 under 0.4 asymmetric noise and 0.2 symmetric noise with imbalanced data.
        Overall, each component improves the performance compared to just naively optimising the weight in \cref{eq:gradient_w}.  
        Adding $\mathsf{Info}(.)$ and  $\mathsf{Clean}(.)$  improves the model significantly.
        Naively selecting samples based on \cref{eq:gradient_w}  facilitates the overfiting of the noisy-label samples, leading to confirmation bias. 
        To mitigate this problem, $\mathsf{Clean}(.)$ limits the noise in the clean validation set, while $\mathsf{Info}(.)$ prevents the gradient to go toward a single wrong direction. 
        
       
           
           
           
           
               
               
               
               
               
                   
               
               
               
               
               
                
               
               
                
                
               
               
                
               
               
                
                
               
               
               
           
           
           
               
                
               
               
               
               
                   
               
               
               
               
               
                
               
               
                
                
               
                
                
               
               
                
                
               
                 
               
           
           
           
       
        
        \begin{table}[t]
            \centering
            \caption{Test accuracy (\%) on CIFAR10 and CIFAR100 under asymmetric and imbalanced noisy-label problems. The $1^{st}$ row shows the results of the optimisation of the weight (col. \textbf{Weight in \cref{eq:gradient_w}})   instead of \cref{eq:select_validation_set}. The  $2^{nd}$ row shows the results of optimising the lower part of \cref{eq:select_validation_set} (col. \textbf{$\mathsf{Info}(.)$ Only}) without the upper part of~\cref{eq:select_validation_set}  $\mathsf{Clean}(.)$. The last row (\textbf{Whole~\cref{eq:select_validation_set}}) shows our final model result.}
            \label{tab:ablatant}
           
               
               
               
               
               
                   
               
               
               
               
               
                
               
               
                
                
               
               
                
               
               
                
                
               
               
               
           
            \scalebox{0.75}{
            \begin{tabular}{ c c c c c c c c}
                \toprule
                 \multirow{4}{*}{\bfseries \shortstack[l]{Weight in \\ \cref{eq:gradient_w}}} & \multirow{4}{*}{\bfseries \shortstack[l]{$\mathsf{Info}(.)$ \\ Only}} & \multirow{4}{*}{\bfseries \shortstack[l]{Whole \\ \cref{eq:select_validation_set}}} &  \multicolumn{2}{c}{\bfseries 0.4 Asymmetric} & \multicolumn{3}{c}{\bfseries 0.2 Uniform}  \\
                \cmidrule{4-8} 
                & &  &  \multicolumn{2}{c}{Cifar10} & \multicolumn{3}{c}{Cifar10}  \\
                \cmidrule{4-8} 
                & & & WRN & RN29 & RN32 & RN32 & RN32  \\
                \cmidrule{1-8} 
                \multicolumn{3}{c}{\bfseries Imb. Ratio} & 1 & 1 & 10 & 50 & 200 \\
                   
                \midrule
               
               
               
                \checkmark &  & & 91.0 &56.6 & 68.8 & 37.6 & 23.4   \\ 
                
               
               
                
                
               
                 & \checkmark & & 92.1 & 89.3  & 89.0 & 79.1  & 65.9  \\
                
               
               
                
                
               
                  &  & \checkmark & 94.1 &89.8 & 90.1 & 80.1 & 66.6  \\
                \bottomrule
            \end{tabular}
            }
           
        \end{table}
        
        We also compare the validation set built with~\cref{eq:select_validation_set} with sets built with random sampling and most confident sampling based on the highest confidence score.
       
         \cref{fig:selection_mostconfident} shows that most confident sampling shows inferior results compared to random sampling, but our method to build the validation set shows the best results.
        
        
       
        
        Traditional meta-learning approaches \cite{L2LWS,MetaWeightNetLA} always keep the clean validation set separate from the training set, while our method iteratively extracts $D^{(v)}$ from the training set.
        It can be argued that this non-separation of the training and validation sets can cause confirmation bias to happen during training. 
        Hence, we tested our approach in a scenario where the candidate samples to form the validation set is always separate from the training set during training.
        However, results showed 
        that such separate validation set causes a $2\%$ drop in accuracy, on average.
        This can be explained by the smaller size of the training set and the restriction in potential choices for validation samples.
       
        
        
        \begin{figure}[t]
            \centering
            \includegraphics[width = 0.99\linewidth]{img/ablatant_study_6.pdf}
            \caption{Accuracy (\%) of our INOLML using different sample selection methods.
            } \label{fig:selection_mostconfident}
           
        \end{figure}
        
        
       
       
            
            A final discussion point is the time needed to run our approach. The Distill model takes around 5 and 29 hours to train the Resnet29 and WideResnet28-10 models, respectively. When integrating our method with Distill, training  takes around 5.5 hours on Resnet29 and 31 hours on WideResnet28-10. Hence, our algorithm adds an 10\% traning time overhead. Experiments are conducted on a single NVIDIA V100 GPU.
     
\section{Conclusion}
\label{sec:conclusion}
 
 We presented a novel meta-learning approach, called INOLML, that automatically and progressively selects a pseudo clean validation set from a noisily-labelled training set. 
 This selection is based on our proposed validation set utility that takes into account sample informativeness, class distribution balance, and label correctness.

 Our proposed method is more effective and practical than prior  meta-learning approaches since we do not require manually-labelled samples to include in the validation set. 

 Compared with other meta-learning approaches that do not require a manually labelled validation set, e.g. FSR or FaMUS, our method is demonstrated to be more robust to high noise rate problems and to achieve state-of-the-art results on several synthetic and realistic label noise benchmarks.
 
 
 A limitation of our approach is that the model can suffer from confirmation bias as it is based on a single model. 
 As future work, we will tackle this problem by incorporating co-training in our meta-learning algorithm. 

 Another limitation is the greedy and complex bi-level optimisation to form the validation set in~\cref{eq:select_validation_set}, which can be improved in two ways: 1) the complexity can be reduced by replacing the bi-level optimisation by a single-level optimisation, and 2) the greedy strategy can be replaced by a relaxation method to solve the combinatorial optimisation problem.
 Additionally, optimising the clean validation set once per epoch is not ideal since the validation set can be outdated by the end of epoch. This issue will be addressed by updating the clean validation set more regularly.
 
 
    
    
    
   

\section{Related Work}
\label{sec:related_work}
    We review methods that can deal with imbalanced noisy-label learning, focusing on meta-learning approaches.
    
    
   
    \subsection{Noisy-label Learning}
    \label{sec:related_work_noisy_label_learning}
        \vspace{-0.5em}
        Current noisy-label learning methods can rely on many strategies, such as: robust loss functions~\cite{wang2019imae, wang2019symmetric, liu2020early}, ensemble learning \cite{miao2015rboost}, student-teacher model~\cite{tarvainen2017mean}, label cleaning \cite{yuan2018iterative, jaehwan2019photometric}, 
        co-teaching~\cite{li2020dividemix, MentorNet, malach2017decoupling, han2018co, yu2019does}, dimensionality reduction~\cite{ma2018dimensionality}, iterative label correction~\cite{zhang2021learning}, 
        semi-supervised learning~\cite{ortego2019towards, li2020dividemix, ortego2021multi}, 
        meta-learning ~\cite{han2018pumpout,sun2021learning,MentorNet,L2W,Distill_noise,FSR,Famus,MetaWeightNetLA,Deep_medical,MetaSL,LearningSL}, 
        and hybrid methods~\cite{yu2018learning,kim2019nlnl,zhang2019metacleaner,SELF,jiang2020beyond}. Usually,  most of the methods above assume that the training set has a balanced distribution of samples per class, except for the meta-learning approaches~\cite{MentorNet,L2W,Distill_noise,FSR,Famus,MetaWeightNetLA,Deep_medical,MetaSL,LearningSL} that not only address the noisy-label problem, but also the learning with an imbalanced training set.
        
       
       
        Meta learning is a versatile solution for  many problems (few-shot learning, reinforcement learning, etc.) that optimises  meta-parameters in order to benefit the training process.
       
        In noisy-label meta learning papers~\cite{MentorNet, L2W, Distill_noise, FSR, Famus, MetaWeightNetLA, Deep_medical, MetaSL, LearningSL}, the meta parameters consist of a weight for each training sample~\cite{FSR,Distill_noise}, and the meta learning methods optimise the model based on a weighted cross entropy loss that automatically downweights noisy samples and upweights clean samples.
       
        For example, L2LWS~\cite{L2LWS} and CWS~\cite{CWS} comprise a target deep neural network (DNN) and a meta-DNN that is pre-trained on a small clean validation dataset 
        to re-weight the training samples to model the target DNN.
        Automatic reweighting~\cite{L2W}  weights training samples based on the performance of one-step-ahead model on the validation set. 
        Except for recent methods~\cite{FSR,Famus}, meta-learning approaches require a clean validation set that can be expensive to acquire or unavailable in real world scenarios. Therefore, similarly to~\cite{FSR,Famus}, we focus on the development of an approach that can automatically build a clean validation set, but unlike them, we propose an approach that is motivated by the meta-learning algorithm.


    \subsection{Imbalanced Learning}
    \label{sec:related_work_imbalanced_learning}
        Imbalance learning is another challenging classification problem that is commonly present in real-world datasets, 
        where a small portion of majority classes have a massive amount of training samples, and minority classes only have a few training samples~\cite{longtail_survey}. 
        This can easily result in a biased model that shows good accuracy for majority classes, but poor performance for the minority ones. 
        To address this problem, 
        many imbalanced learning methods have been proposed~\cite{transfer_imb,transfer_imb_2,class_imb,class_imb_2,resample_2}, where the main techniques are~\cite{longtail_survey}: transfer learning~\cite{transfer_imb_2,transfer_imb}, classifier design~\cite{class_imb,class_imb_2}, re-sampling (e.g., meta-learning)~\cite{resample_2}, decoupled training~\cite{decouple,decouple_2}, ensemble learning~\cite{BBN,ensemble_imb_1}, cost-sensitive learning~\cite{cost_sensitive_2,cost_sensitive_3,cost_sensitive}, data augmentation~\cite{data_aug,data_aug_2}, logit adjustment~\cite{logit_adjust,logit_adjust_2} and representation learning~\cite{represent_2,represent}.
       
        Unfortunately, existing methods designed to learn from long-tailed class distributions assume the labels to be clean, making their performance unclear in a more realistic scenario where the datasets are also noisy. 
        
    \subsection{Noisy-label and Imbalanced Learning}
        Most of the papers listed in Sections \ref{sec:related_work_noisy_label_learning} and \ref{sec:related_work_imbalanced_learning} studied noisy label and imbalance learning problems as separate problems, except FSR~\cite{FSR} -- a recent meta-learning approach that aims to solve both problems with meta-learning. The presence of label noise in imbalanced datasets has also been considered by  non meta-learning approaches~\cite{Imbalance_noisy,ROLT,Karthik2021LearningFL}, but they either have different setups or achieve inferior results compared with recently proposed meta-learning approaches. As mentioned in Sec.~\ref{sec:introduction},
        the validation set plays a central role in meta-learning, but we are not aware of papers that study how to maximise its utility during optimisation.
        Classic meta-learning approaches~\cite{L2W,MetaWeightNetLA} relies on a random sample selection and manually labelling approach that will likely result in a sub-optimal validation set. 
        In addition, the fact that such manually-curated validation set is fixed for the whole training process may hinder the generalisation of the model. 
        Recent meta-learning approaches try to automatically build a validation set that varies throughout the training process. For instance, FaMUS~\cite{Famus} selects training samples with low losses to form the validation set, while FSR~\cite{FSR} chooses samples that can be well optimized after a training iteration to build the validation set. Such methods, however, form a validation set based on heuristics that are not directly related to the meta-learning optimisation, which is the problem being studied in this paper.

       
\section{Introduction }
\label{sec:introduction}
   
    Within the past decade, there have been great advancements in visual classification \cite{MentorNet,han2018co,Wang2018IterativeLW}, object detection~\cite{object_detection_1,object_detection_2,object_detection_3} and segmentation~\cite{segman_1,segman_2} thanks in part to deep learning models. 
    The functionality of these models partly depends on large training sets containing samples that have been correctly labelled and that are well-balanced among the classes.
    The difficulty in obtaining such training sets is motivating researchers to develop methods that can work with less well curated data sets~\cite{clothing1M,Webvision}.
    Unfortunately, such poorly curated datasets are more likely to contain label noise and imbalanced class distribution.


    In the literature, the problems of imbalanced learning and noisy-label learning are generally treated separately. 
   
    While noisy label methods are based on robust loss functions~\cite{Robust_loss,wang2019symmetric}, label cleaning~\cite{jaehwan2019photometric,yuan2018iterative}, meta-learning~\cite{L2W,han2018pumpout}, ensemble learning~\cite{miao2015rboost}, and other methods~\cite{kim2019nlnl,zhang2019metacleaner}, imbalanced learning approaches are based on meta-learning~\cite{L2W,han2018pumpout,FSR}, transfer learning~\cite{transfer_imb_2,transfer_imb}, classifier design~\cite{class_imb_2,class_imb}, re-sampling~\cite{resample_2}, and etc. Among those approaches, meta-learning based methods~\cite{MentorNet, L2W, Distill_noise, FSR, Famus, MetaWeightNetLA, Deep_medical, MetaSL, LearningSL} can address both noisy-label and imbalanced learning problems.
    
    
    
   
    Meta-learning is often formulated as a bi-level optimisation, where the upper level estimates the meta parameters using the validation set, and the lower level trains a classifier using the training set and the estimated meta-parameters, where the validation set is commonly built by randomly selecting and manually labelling training samples.
    However, the process of building these validation sets scales poorly with the number of classes, and the random selection may not pick the most informative samples. These issues have motivated the design of ad-hoc methods to build the validation set~\cite{FSR,Famus}. Unfortunately, their results are not as competitive as approaches that rely on manually-curated validation sets.
    This issue may be due to a shortcoming in their proposed heuristics~\cite{FSR,Famus}, which characterises balanced distribution and label cleanliness but ignores the informativeness for the meta-learning algorithm.

   

    
    \begin{figure*}[t!]
        \centering
        \includegraphics[width=0.75\textwidth]{motivation_dung.png} 
        \caption{Main stages of INOLML: 1) classify the noisy-label samples from $\mathcal{D}$ into $\mathcal{D}^{(c)}$ (with samples that are likely to have clean labels) and  $\mathcal{D}^{(n)}$ (samples likely to have noisy labels); 2) build a validation set $\mathcal{D}^{(v)}$ containing samples that are informative (from a meta-learning perspective), balanced and with a high likelihood of containing clean labels tested with the moving average robust labeller, where the training set $\mathcal{D}^{(t)} = \mathcal{D}^{(c)}\setminus \mathcal{D}^{(v)}$; and 3) train the meta-learning classifier with $\mathcal{D}^{(t)}$ and $\mathcal{D}^{(v)}$. 
        These three steps are iterated during training.
        }
        \label{fig:motivation}
       
    \end{figure*}
    
     In this paper, we propose a new imbalanced noisy-label meta-learning (INOLML) method that automatically builds a validation set by maximising its utility in terms of sample informativeness, class distribution balance, and label correctness. The central contribution of the paper is the definition of the validation set utility criteria, which is motivated by the bi-level optimisation meta-learning algorithm.
     The proposed method, depicted in \cref{fig:motivation}, consists of an iterative 3-step approach, namely: 1) 
     pseudo-clean sample detection and robust labelling from the noisy training set; 2) validation set formation from the robustly labelled pseudo-clean set in step (1), using the proposed utility criteria; and 3) meta learning using the validation set from step (2). The main contributions of our paper can be summarised as follows:
     \begin{itemize}
        \item A new method to build the meta-learning validation set by maximising its utility for sample informativeness, class distribution balance, and label correctness;
         \item An innovative meta-learning algorithm (\cref{fig:motivation}), comprising the steps:
         1) detection and robust labelling of pseudo-clean samples from the noisy training set; 2)
         formation of the validation set using the proposed utility criteria; and 3) meta learning using the validation set from step (2). 
        
        
     \end{itemize}
     With the two technical  contributions above, 
    
    
     our method shows improvements over previous meta-learning approaches on imbalanced  noisy-label learning benchmarks. 
     In balanced noisy-label benchmarks, our method is competitive or better than the state-of-the-art.
     

\section{Method}
\label{sec:method}
   
    The initial training set 
    is defined as $\mathcal{D}=\{ (\mathbf{x}_{i},\mathbf{y}_{i})\}_{i=1}^{|\mathcal{D}|}$, with $\mathbf{x}_{i} \in \mathcal{X} \subseteq \mathbb{R}^{H \times W \times R}$ representing an image of size  $H \times W$ pixels and $R$ colour channels, and \(\mathbf{y}_{i} \in \mathcal{Y} = \{\mathbf{v}: \mathbf{v} \in \{0, 1\}^{C} \text{ and } \sum_{k=1}^{C} \mathbf{v}(k) = 1 \}\) being the noisy one-hot and $C$ denoting the number of classes~\cite{hospedales2020meta}.
    The classification model is represented by  \(f_{\theta}: \mathcal{X} \to \Delta_{C - 1}\) parameterised by $\theta \in \Theta$, with
   
    the \(C-1\) probability simplex \(\Delta_{C - 1} = \{\mathbf{p}: \mathbf{p} \in [0, 1]^{C} \text{ and } \sum_{k = 1}^{C} \mathbf{p}(k) = 1\}\).
   
    
    
   
    
   
    
    The proposed INOLML follows a bi-level optimisation~\cite{L2W,Distill_noise} that relies on the meta-parameter $\omega = \{\omega_i\}_{i=1}^{|\mathcal{D}^{(t)}|}$ ($\omega_i \ge 0$) that weights the samples in the training set \(\mathcal{D}^{(t)}\) based on their utility regarding informativeness and label cleanliness, and $\lambda = \{ \lambda_i \}_{i=1}^{|\mathcal{D}^{(t)}|}$ ($\lambda_i \in [0,1]$) that weights the contribution of model prediction in the pseudo-label estimation, as in $\hat{\mathbf{y}}_i(\lambda_i) = \lambda_i \mathbf{y}_i + (1-\lambda_i)f_{\theta}(\mathbf{x}_i)$. The meta-learning optimisation is defined by:
    \begin{equation}
    \scalebox{0.9}{$
        \begin{aligned}[b]
            & \omega^{*},\lambda^{*} =  \arg\min_{\omega,\lambda} \frac{1}{|\mathcal{D}^{(v)}|} \sum_{(\mathbf{x}_j, \mathbf{y}_j) \in \mathcal{D}^{(v)}}   \ell^{(v)} \left( \mathbf{x}_j, \mathbf{y}_j; \theta^{*} \left( \omega,\lambda \right) \right) \\
            \text{s.t.: } &
            \theta^{*}(\omega,\lambda) = \arg\min_{\theta}             \frac{1}{\abs{\mathcal{D}^{(t)}}}
            \sum_{( \mathbf{x}_i, \mathbf{y}_i ) \in \mathcal{D}^{(t)}}
             \omega_i \, \ell^{(t)} \left(\mathbf{x}_i, \hat{\mathbf{y}}(\lambda_i); \theta \right),
        \end{aligned}
        $}
        \label{eq:meta_learning_bilevel_optimisation}
    \end{equation}
    where:
   
    \(\ell^{(v)}(\mathbf{x}_j,\mathbf{y}_j;\theta^{*}(\omega,\lambda)) = \ell_{\mathrm{CE}}(\mathbf{y}_j, f_{\theta^{*}(\omega,\lambda)}(\mathbf{x}_j))\) 
    is the cross-entropy (CE) loss between the label \(\mathbf{y}_{j}\) and model prediction,
   
    $\ell^{(t)} \left(\mathbf{x}_i, \hat{\mathbf{y}}(\lambda_i); \theta \right)$ is defined below in~\eqref{eq:training_loss_model_parameters}. The validation set $\mathcal{D}^{(v)}$ is obtained by:
    \begin{equation}
        \mathcal{D}^{(v)} = \mathsf{MaxUtility}\left (\mathcal{D}^{(c)}\right ),
        \label{eq:MaxUtility}
    \end{equation}
    which depends on the pseudo-clean set $\mathcal{D}^{(c)}$ obtained from:
    \begin{equation}
        \mathcal{D}^{(c)}=\mathsf{PseudoCleanDetector}\left (\mathcal{D} \right ).
        \label{eq:PseudoCleanDetector}
    \end{equation}
   
    
    
    Our main contribution is the definition of the utility criteria in $\mathsf{MaxUtility} (\mathcal{D}^{(c)})$ in~\eqref{eq:MaxUtility} to select the validation set \(\mathcal{D}^{(v)}\) and training set \(\mathcal{D}^{(t)}\), where $\mathcal{D}^{(v)} \subset \mathcal{D}^{(c)}$, with \(\mathcal{D}^{(t)} \cap \mathcal{D}^{(v)} = \varnothing\) and \(\mathcal{D}^{(t)} \cup \mathcal{D}^{(v)} = \mathcal{D} \).
    The validation set $\mathcal{D}^{(v)}$ has a balanced distribution of samples per class, contains samples that are informative for the meta-learning optimisation in~\eqref{eq:meta_learning_bilevel_optimisation} and are likely to have clean labels.
    
    
    The pseudo-clean sample  set $\mathcal{D}^{(c)}$ is estimated with a noisy-label classifier, represented by \(\mathsf{PseudoCleanDetector}(.)\) shown in \eqref{eq:PseudoCleanDetector}. Such classifier selects pseudo-clean samples based on the small CE loss hypothesis~\cite{han2018co, li2020dividemix} where the loss is computed between the labels in $\mathcal{D}$ and the predictions by $f_{\theta}(.)$. The remaining samples form $\mathcal{D}^{(n)}$, with $\mathcal{D}^{(c)} \cup \mathcal{D}^{(n)} = \mathcal{D}$ and $\mathcal{D}^{(c)} \cap \mathcal{D}^{(n)} = \varnothing$. These sets
   
    are regularly updated during training.
    The initial pseudo-clean set at the first training iteration is estimated from the model $f_{\theta}(.)$ trained with early-stopping.
   
   
   
   
   
    In the following subsections, we describe how to select a validation set that maximises its utility in terms of informativeness, label cleanliness and class distribution balance.
    
    \subsection{Maximising the Utility of the Validation Set}
    \label{sec:maximising_utility_validation_set}
    
    The maximisation of the utility of the validation set is motivated by the
    bi-level optimisation
    in~\eqref{eq:meta_learning_bilevel_optimisation}, where we focus on the weighting of each training sample, represented by $\omega_i$, which estimates the importance of that sample in the training process.
   
   
    The optimisation in \eqref{eq:meta_learning_bilevel_optimisation} is solved by iterating the following 2 steps. In the first step, the locally-optimal model parameter \(\theta^{*}(\omega, \lambda)\) in the lower-level is obtained by applying  stochastic gradient descent (SGD) on the training set \(\mathcal{D}^{(t)}\) with each step defined by:
    \begin{equation}
    \scalebox{0.85}{$
        \begin{aligned}[b]
            \hat{\theta}(\omega,\lambda) = & \theta(\omega,\lambda) - \\
            & \eta_{\theta} \grad_{\theta} \eval{ \left (\frac{1}{|\mathcal{D}^{(t)}|} \sum_{( \mathbf{x}_{i}, \mathbf{y}_{i} ) \in \mathcal{D}^{(t)}} \omega_i \, \ell^{(t)} (\mathbf{x}_{i}, \hat{\mathbf{y}}_{i}(\lambda_i); \theta ) \right) }_{\theta=\theta(\omega,\lambda)}. 
            \end{aligned}
            $}
        \label{eq:update_theta}
    \end{equation}
   
   


    In the second step, the meta-parameters, \(\omega\) and \(\lambda\), in the upper-level, are updated by applying one SGD step on the validation set \(\mathcal{D}^{(v)}\). 
    For \(\omega\), the update is defined as:
    \begin{equation}
    \scalebox{0.82}{$
        \begin{aligned}[b]
        & {\omega}^*_i = \max\left(0,  - \frac{\eta_{\omega} }{|\mathcal{D}^{(v)}|} \sum_{(\mathbf{x}_{j}, \mathbf{y}_{j}) \in \mathcal{D}^{(v)}} \eval{\pdv{}{\omega_{i}} \ell^{(v)} ( \mathbf{x}_{j}, \mathbf{y}_{j}; \theta^{*} ( \omega,\lambda ) ) }_{\omega_i=0}
        \right), \\
       
       
        \end{aligned}
        $}
        \label{eq:update_w}
    \end{equation}
    and the update for \(\lambda\) is defined below in~\eqref{eq:update_lambda}.
    The obtained meta-parameters are then used in the next bi-level optimisation iteration.
    

    According to \cite{L2W}, the gradient w.r.t. \(\omega\) is expressed as:
    \begin{equation}
    \scalebox{0.9}{$
        \begin{aligned}[b]
             \sum_{(\mathbf{x}_{j}, \mathbf{y}_{j}) \in \mathcal{D}^{(v)}} & \eval{ \pdv{}{\omega_{i}} \ell^{(v)} ( \mathbf{x}_{j}, \mathbf{y}_{j}; \theta^{*} ( \omega,\lambda ) ) }_{\omega_i=0} \propto \\
            &-\sum_{(\mathbf{x}_{j}, \mathbf{y}_{j}) \in \mathcal{D}^{(v)}} \sum_{l=1}^{L} ( {\mathbf{z}^{(v)}_{j,l-1}}^{\top} \mathbf{z}^{(t)}_{i,l-1} ) ({\mathbf{g}^{(v)}_{j,l}}^{\top}\mathbf{g}^{(t)}_{i,l} ),
        \end{aligned}
        $}
        \label{eq:gradient_w}
    \end{equation}
   
    where $\mathbf{z}^{(v)}_{j,l-1}$ denotes the feature from 
    validation image $\mathbf{x}_{j}$
    to be processed by layer $l$ of the model (similarly for the training image feature $\mathbf{z}^{(t)}_{i,l-1}$), and 
    $\mathbf{g}^{(v)}_{j,l}$ represents gradient from layer $l$ for the validation image $\mathbf{x}_{j}$ (similarly for the training image gradient $\mathbf{g}^{(t)}_{i,l}$).  Hence, the weight of a training sample is high if both its feature and gradient are similar to the feature and gradient of at least one of the validation samples; otherwise, the weight is low. 
    
    Therefore, a validation set that maximises the weight of samples in the training set maximises its utility for the meta-learning optimisation. This observation is at the crux of our validation sample selection approach, where we first form a pseudo-clean set from the training set and then search within that pseudo-clean set to form a validation set that is balanced and maximises the sum in~\eqref{eq:gradient_w}.
    The validation set $\mathcal{D}^{(v)} \subset \mathcal{D}^{(c)}$ is built
    with the function  $\mathsf{MaxUtility(.)}$ from~\eqref{eq:MaxUtility} 
    with the following bi-level optimisation:
        \begin{equation}
            \begin{aligned}[b]
            & \mathcal{D}^{(v)}  = \arg\max_{\substack{\widehat{\mathcal{D}}^{(v)} \subset \widetilde{\mathcal{D}}^{(v)}\\ |\widehat{\mathcal{D}}^{(v)}|=M\times C}}
            \mathsf{Clean} \left( \widehat{\mathcal{D}}^{(v)},\mathcal{D}^{(c)} \right)\\
            & \text{s.t.: }
             \widetilde{\mathcal{D}}^{(v)}  = \arg\max_{\substack{\bar{\mathcal{D}}^{(v)} \subset {\mathcal{D}}^{(c)}\\ |\bar{\mathcal{D}}^{(v)}|=K\times C}}
             \mathsf{Info} \left( \bar{\mathcal{D}}^{(v)},\mathcal{D}^{(c)} \right).
            \end{aligned}
            \label{eq:select_validation_set}
        \end{equation}
        The function $\mathsf{Info}(.)$ in the lower-level of \eqref{eq:select_validation_set} is defined as:
        \begin{equation}
            \mathsf{Info}(\bar{\mathcal{D}}^{(v)},\mathcal{D}^{(c)}) = \sum_{\substack{(\mathbf{x}_i,\mathbf{y}_i) \in \mathcal{D}^{(c)}\setminus\bar{\mathcal{D}}^{(v)}}} \max_{\substack{(\mathbf{x}_j,\mathbf{y}_j) \in \bar{\mathcal{D}}^{(v)} \\ \mathbf{y}_j = \mathbf{y}_i}} \iota(\mathbf{x}_i,\mathbf{x}_j), 
            \label{eq:informativeness}
        \end{equation}
        with
        \begin{equation}
            \iota(\mathbf{x}_i,\mathbf{x}_j) =\sum_{l=1}^L (\mathbf{z}_{j,l-1}^{\top}\mathbf{z}_{i,l-1}) (\mathbf{g}_{j,l}^{\top}\mathbf{g}_{i,l}),
            \label{eq:iota}
        \end{equation}
        where, similarly to~\eqref{eq:gradient_w}, $\mathbf{z}_{j,l-1}$ is the image feature input to layer $l$ from $\mathbf{x}_j$ (same for $\mathbf{z}_{i,l-1}$ from $\mathbf{x}_i$), and $\mathbf{g}_{j,l}$ denotes the validation image gradient of layer $l$ from $\mathbf{x}_j$ (same for $\mathbf{g}_{i,l}$ from $\mathbf{x}_i$). 
        Note that the function $\iota(.)$ defined in~\eqref{eq:iota} is the weight defined in~\eqref{eq:gradient_w} between the training sample $(\mathbf{x}_i,\mathbf{y}_i)$ and the validation sample $(\mathbf{x}_j,\mathbf{y}_j)$, or the \say{information} that $(\mathbf{x}_i,\mathbf{y}_i)$ can get from $(\mathbf{x}_j,\mathbf{y}_j)$. Intuitively, the lower-level summation in~\eqref{eq:select_validation_set} (and in particular \eqref{eq:informativeness}) is designed to form a candidate balanced set $\widetilde{\mathcal{D}}^{(v)}$ by maximising the maximum \say{information content} that the pseudo-clean samples from $\mathcal{D}^{(c)} \setminus \widetilde{\mathcal{D}}^{(v)}$ can get from the samples in  $\widetilde{\mathcal{D}}^{(v)}$. 
        The reason we maximise the maximum instead of the average \say{information content} is to guarantee that each clean training sample get upweighted by at least one clean validation sample.
        Unfortunately, the samples selected to be in $\widetilde{\mathcal{D}}^{(v)}$ can still have noisy labels since $D^{(c)}$ is not completely clean and the function \(\mathsf{Info}(.)\) tends to 
        return high values if
        samples in $\widetilde{\mathcal{D}}^{(v)}$  have low confidence logit scores and high gradient values.
        Simply filtering out samples with higher gradient will force the validation set to contain samples that are more likely to be clean, but less likely to be informative.
        Therefore, we aim to identify samples that are likely to have clean labels without relying on their prediction logits. 

        To search for clean samples in \(\widetilde{D}^{(v)}\), we observe that 
        the samples from this set are more likely to be clean when they have higher similarity with other samples belonging to the same class. Given this observation, we therefore propose a heuristic based on the cosine similarity between the sample of interest and other samples of the same class in the pseudo-clean set \(\mathcal{D}^{(c)}\). The heuristic is represented by the function \(\mathsf{Clean}(.)\), which is defined as follows:
        \begin{equation}
            \mathsf{Clean} \left( \widehat{\mathcal{D}}^{(v)},\mathcal{D}^{(c)} \right) =  \sum_{\substack{(\mathbf{x}_j,\mathbf{y}_j) \in \widehat{\mathcal{D}}^{(v)}\\
            (\mathbf{x}_i,\mathbf{y}_i) \in \mathcal{D}^{(c)}\setminus\widehat{\mathcal{D}}^{(v)} \\ \mathbf{y}_i = \mathbf{y}_j}}
             \sum_{l=1}^L \left(\mathbf{z}_{j,l-1}^{\top}\mathbf{z}_{i,l-1} \right).
             \label{eq:clean}
        \end{equation}
        We also impose a constraint that selects \(M\) samples for each of the \(C\) classes with \(M \ll K\) as shown in the upper-level of \eqref{eq:select_validation_set} to obtain a balanced validation subset \(\mathcal{D}^{(v)}\).
        
        Given that both optimisations in~\eqref{eq:select_validation_set} consist of combinatorial problems, we resort to a greedy approach that loops through the classes and sequentially selects \(M\) and \(K\) samples for each class (for the upper and lower optimisation, respectively) that maximise the respective objective function. As the solution needs to iterate through all layers of a neural network of interest, the calculation of gradient in \eqref{eq:gradient_w} and the optimisation in \eqref{eq:select_validation_set} might be expensive, especially for large-scale deep neural networks.
        However, according to Zhang et al.~\cite{FSR}, the weights of training samples in meta-learning  mostly depend  on the last layer of the  model. Hence, we use only on the last layer $L$ of the model for~\eqref{eq:gradient_w} and \eqref{eq:select_validation_set} to reduce the computational cost.
       




    \subsection{Training Procedure}
    \label{sec:training_procedure}
    
    Our training procedure follows the 3-step iterative approach depicted in \cref{fig:motivation}, where step 1 (pseudo-clean label detector) and step 2 (maximise the utility of the validation set) have been explained in~\cref{sec:maximising_utility_validation_set}.
    Step 3 (meta-learning) is based on the optimisation in \eqref{eq:meta_learning_bilevel_optimisation}, where our training loss $\ell(.)$  follows the one defined in~\cite{Distill_noise}.
    
    
    To optimise \eqref{eq:meta_learning_bilevel_optimisation}, we first estimate $\omega^{*}$ with~\eqref{eq:update_w} and
    $\lambda^{*}$ (i.e., the pseudo-labelling parameter defined in~\eqref{eq:meta_learning_bilevel_optimisation}) with~\cite{Distill_noise}:
    \begin{equation}
    \scalebox{0.9}{$
        \begin{aligned}[b]
        & {\lambda}^*_i =  \left [ \mathrm{sign} \left (  \sum_{(\mathbf{x}_{j}, \mathbf{y}_{j}) \in \mathcal{D}^{(v)}} \pdv{}{\lambda_{i}} \ell^{(v)} ( \mathbf{x}_{j}, \mathbf{y}_{j}; \theta^{*} ( \omega,\lambda ) ) 
        \right ) \right ]_{+},
        \end{aligned}
        $}
        \label{eq:update_lambda}
    \end{equation}
    where \(\ell^{(v)}(\mathbf{x}_j,\mathbf{y}_j;\theta^{*}(\omega,\lambda))\) is defined in~\eqref{eq:meta_learning_bilevel_optimisation}.
    
    
    After estimating $\omega^{*}$ and $\lambda^{*}$, we optimise the lower level  of~\eqref{eq:meta_learning_bilevel_optimisation} to estimate the model parameter
    using the following loss function~\cite{Distill_noise}:
    \begin{equation}
    \scalebox{0.85}{$
         \begin{aligned}[b]
         \ell^{(t)}(\mathbf{x}_i,\mathbf{y}_i;\theta) = & \omega^{*}_i\ell_{\mathrm{CE}}(\widehat{\mathbf{y}}_i(\lambda_0), f_{\theta}(\mathbf{x}_i))+ \frac{1}{B}\ell_{\mathrm{CE}}(\mathbf{y}_i^{*}(\lambda^*_i), f_{\theta}(\mathbf{x}_i)) + \\
        & p \times \ell_{\mathrm{CE}}(\mathbf{y}_i^{\beta},f_{\theta}(\mathbf{x}_i^{\beta})) +  k \times \ell_{\mathrm{KL}}(f_{\theta}(\mathbf{x}_i),f_{\theta}(\hat{\mathbf{x}}_i)),
        \end{aligned}
        $}
        \label{eq:training_loss_model_parameters}
    \end{equation}
    where $\widehat{\mathbf{y}}_i(\lambda_0)$ is a pseudo-label, defined as in \eqref{eq:meta_learning_bilevel_optimisation}, with a fixed weight $\lambda_{0} = 0.9$, $\mathbf{y}_i^{*}(\lambda^*_i)=\mathbf{y}_i$ if $\lambda^*_i>0$, $\mathbf{y}_i^{*}=f_{\theta}(\mathbf{x}_i)$ if $\lambda^*_i\le 0$, $\mathbf{y}_i^{\beta}$ and $\mathbf{x}_i^{\beta}$ are obtained via the mixup operator~\cite{zhang2018mixup} using the training and validation sets, $\ell_{\mathrm{KL}}(.,.)$ represents the Kullback-Leibler (KL) divergence~\cite{kullback1951information} between the model response for training image $\mathbf{x}_i$ and its data augmented version $\hat{\mathbf{x}}_i$, \(p\) and \(k\) are hyper-parameters, and \(B\) is the batch size. 
    
    The effectiveness of the optimisation in~\eqref{eq:select_validation_set} depends on the actual (hidden) proportion of clean samples in the pseudo clean set $D^{(c)}$, while the efficiency depends on the size of $D^{(c)}$.
    Hence, to reduce computational cost, the selection of the validation set in~\eqref{eq:select_validation_set} uses a subset \(\widetilde{D}^{(c)} \subset \{(\mathbf{x}_{i}, \mathbf{y}_{i}): (\mathbf{x}_{i}, \mathbf{y}_{i}) \in D^{(c)} \wedge \arg\max_{k\in\{1,...,C\}}\mathbf{y}_{i}(k) = \arg\max_{k\in\{1,...,C\}} \tilde{\mathbf{y}}_{i}(k)\}\). This subset contains $N$ randomly-selected samples \((\mathbf{x}_{i}, \mathbf{y}_{i})\) of each class in $D^{(c)}$ that have their observed labels $\mathbf{y}_{i}$ consistent with the corresponding moving average robust label computed with the average prediction over the last $E$ epochs, as in \(\tilde{\mathbf{y}}_i = \kappa \tilde{\mathbf{y}}_i +  \nicefrac{(1-\kappa)}{E} \sum_{e=1}^{E} f_{\theta}(\mathbf{x}_i)\), with \(\kappa \in [0, 1]\) being a hyper-parameter. The details of the training process are in \cref{algorithm:training_procedure} of \cref{sec:algorithm}. 
    
         
        
\section{Experiment and Analysis}
\label{sec:experiments_and_analysis}

We evaluate our method INOLML on four datasets: CIFAR10, CIFAR100~\cite{Cifar_dataset}, WebVision~\cite{Webvision} and Controlled Noisy Web Labels (CNWL)~\cite{Famus} with different noise settings, including symmetric, asymmetric, openset~\cite{OpenSet_noise, Wang2018IterativeLW}, and long-tailed imbalance with and without symmetric noise~\cite{FSR}. For each type of
experiment, we keep the noisy training set the same across all models for a fair comparison.

\subsection{Datasets}

CIFAR10 and CIFAR100 datasets~\cite{Cifar_dataset} contain 50k and 10k images used for training and testing, respectively. Each image has size \(32 \times 32 \) pixels and is labelled as one of 10 or 100 classes. 
WebVision~\cite{Webvision} is a dataset of 2.4 million images crawled from Google and Flickr based on the 1,000 ImageNet classes~\cite{deng2009imagenet}. The dataset is more challenging than CIFAR since it is class-imbalanced and contains real-world noisy labels. Following~\cite{FSR}, we extract a subset that contains the first 50 classes to create the WebVision mini dataset~\cite{MentorNet}. 
CNWL~\cite{jiang2020beyond} is a new benchmark of controlled real-world label noise from the web that contains various noise rates ranging from 0 to 0.8. Following FaMUS~\cite{Famus}, we evaluate the proposed method on Red Mini-ImageNet dataset that consists of 50k training images from 100 classes for training and 5k images for testing.


\subsection{Implementation Details}

For all experiments on CIFAR datasets, except long-tail imbalance, we use the same hyperparameters and network architectures as the Distill model~\cite{Distill_noise}.
We adopt the cosine learning rate decay with warm restarting~\cite{Cosine_lr} and SGD optimiser. For CIFAR datasets, we train WideResnet28-10 with 100k iterations and a batch size of 100. We also train a smaller network (Resnet29) to fairly compare with~\cite{Distill_noise}. For WebVision, we follow FSR~\cite{FSR} and train a single Resnet50 network 
with 1 million iterations and a batch size of 16. For Red mini-ImageNet, we run experiments with 150k iterations and a batch size of 100.
For CNWL, we use a single PreAct Resnet18 network that is similar to previous works~\cite{propmix,ortego2021multi} on this benchmark. For the class imbalance problems, we use the popular Resnet32 model to fairly compare with FaMUS~\cite{Famus} and FSR~\cite{FSR}. We report the prediction accuracy of each experiment on their corresponding testing sets. Please refer to \cref{sec:implementation_details_appendix} for implementation details and hyper-parameters values. 


\begin{table}[t]
    \centering
    \caption{Test accuracy (in \%) of our INOLML and previous methods evaluated on various symmetric noise rates. Methods with superscript \textsuperscript{T} represent meta-learning methods that need clean validation sets. The lower block contains meta-learning methods while the upper block shows methods with SOTA results.}
    \label{tab:table_2}
    \scalebox{0.58}{
    \begin{tabular}{l c | c | c | c | c | c}
    \toprule
    \multirow{2}{*}{\bfseries Method}
   
    & \multicolumn{3}{c}{\bfseries CIFAR10} &  \multicolumn{3}{c}{\bfseries CIFAR100}  \\
    \cmidrule{2-7} 
    & \multicolumn{1}{c}{\bfseries 0.2} & \multicolumn{1}{c}{\bfseries 0.4} & \multicolumn{1}{c|}{\bfseries 0.8} & \multicolumn{1}{c}{\bfseries 0.2} & \multicolumn{1}{c}{\bfseries 0.4} & \multicolumn{1}{c}{\bfseries 0.8} \\ 
    \midrule
   
    GJS\cite{GCE} & 95.3 \(\pm\) 0.2 & 93.6 \(\pm\) 0.2 & 79.1 \(\pm\) 0.3 & 78.1 \(\pm\) 0.3 & 75.7 \(\pm\) 0.3 & 44.5 \(\pm\) 0.5  \\
    DivideMix\cite{lee2019robust}   & 95.7 \(\pm\) 0.0 & - & 92.9 \(\pm\) 0.0 & 76.9 \(\pm\) 0.0 & -  & 59.6 \(\pm\) 0.0 \\
    CRUST\cite{CRUST}   & 91.1 \(\pm\) 0.2 & 89.2 \(\pm\) 0.2& 58.3 \(\pm\) 1.8& - & - & - \\
    PENCIL\cite{PENCIL}   & - & - & - &73.9 \(\pm\) 0.3 & 69.1 \(\pm\) 0.6 & - \\
    ELR\cite{liu2020early}   & 92.1 \(\pm\) 0.4 &91.4 \(\pm\) 0.2& 80.7 \(\pm\) 0.6 & 74.7 \(\pm\) 0.3 & 68.4 \(\pm\) 0.4 & 30.2 \(\pm\)  0.8 \\
    
    \midrule
    
    FaMUS \cite{GCE}  & - & 95.3 \(\pm\) 0.2 & -  & - & 76.0 \(\pm\) 0.2 & -  \\
    Distill\textsuperscript{T}~\cite{Distill_noise} & 96.2 \(\pm\) 0.2 & 95.9 \(\pm\) 0.2 & 93.7 \(\pm\) 0.5 & 81.2 \(\pm\) 0.7 & 80.2 \(\pm\)  0.3 & \(\mathbf{75.5} \pm \mathbf{0.2}\)\\
    MentorNet\textsuperscript{T}\cite{MentorNet}   & 92.0 \(\pm\) 0.0 & 89.0 \(\pm\) 0.0 & 49.0 \(\pm\) 0.0 & 73.0 \(\pm\) 0.0 & 68.0 \(\pm\) 0.0 & 35.0 \(\pm\)  0.0 \\
    L2R\textsuperscript{T}\cite{L2W}   & 90.0 \(\pm\) 0.4 & 86.9 \(\pm\) 0.2 & 73.0 \(\pm\) 0.8 & 67.1 \(\pm\) 0.1 & 61.3 \(\pm\) 2.0 & 35.1 \(\pm\)  1.2 \\
    MWN\textsuperscript{T} \cite{MetaWeightNetLA}  & 90.3 \(\pm\) 0.6 &87.5 \(\pm\) 0.2& - & 64.2 \(\pm\) 0.3&58.6 \(\pm\) 0.5& - \\
    GDW\textsuperscript{T} \cite{GDW}  & - & 88.1 \(\pm\) 0.4 & - & - & 59.8 \(\pm\)1.6 & - \\
    FSR\cite{FSR}   & 95.1 \(\pm\) 0.1 &93.7 \(\pm\) 0.1 &82.8 \(\pm\) 0.3 & 78.7 \(\pm\) 0.2 & 74.2 \(\pm\) 0.4 & 46.7 \(\pm\)  0.8 \\
    \midrule
    \rowcolor{gray!30} \textbf{INOLML} & \(\mathbf{96.9} \pm \mathbf{0.1}\) & \(\mathbf{96.6} \pm \mathbf{0.1}\) & \(\mathbf{95.0} \pm \mathbf{0.2}\) & \(\mathbf{82.0} \pm \mathbf{0.2}\) & \(\mathbf{81.3} \pm \mathbf{0.2}\) & 74.7 \(\pm\) 0.1 \\
    \bottomrule
    \end{tabular}
    }
   
\end{table}

\subsection{Symmetric Noise} 

\cref{tab:table_2} shows the test accuracy of many methods, including meta-learning based approaches that require a clean validation set (indicated with \textsuperscript{T}) and others that automatically build their validation sets, at various level of noise rates ranging from 20\% to 80\%. In general, the proposed method outperforms most of the previous methods, even though we do not require a clean validation set. The slightly lower performance than Distill on CIFAR100 at 80\% noise rate can be explained by Distill's large manually curated clean validation set with 10 clean samples per class. In addition, as shown in~\cref{fig:probe_acc}, at 80\% symmetric noise rate, a significant portion (20\% to 45\%) of our clean validation set contains noisy samples at the final training stages, which deteriorates the efficacy of our approach.
We also carry out additional experiments with different validation set sizes to fairly compare with Distill in \cref{sec:additional_symmetric_noise_results}, in which our method outperforms Distill by 1 to 3\% in majority of scenarios.
Overall, these results show that a pseudo-clean, balanced, and informative validation set, can outperform a randomly-collected clean validation set in most symmetric noise scenarios. Our results also set new state-of-the-art (SOTA) results on the symmetric label noise benchmarks for methods that do not require clean validation set.

\begin{figure}[t]
    \centering
    \begin{subfigure}[t]{0.495 \linewidth}
        \includegraphics[width = \linewidth]{img/probe_acc_2.pdf}
        \caption{Symmetric noise benchmarks}
        \label{fig:probe_acc}
    \end{subfigure}
    \hfill
    \begin{subfigure}[t]{0.495 \linewidth}
        \includegraphics[width = \linewidth]{img/probe_acc_open_2.pdf}
        \caption{Open-set noise benchmarks}
        \label{fig:probe_acc_open}
    \end{subfigure}
   
   
   
   
   
    \caption{Accuracy of the clean validation set \(\mathcal{D}^{(v)}\) as training progresses evaluated on different noise benchmarks.
    }
    \label{fig:accuracy_probe_openset}
   
\end{figure}

        \subsection{Asymmetric Noise}
            We compare our algorithm with  Distill~\cite{Distill_noise}, FSR~\cite{FSR} and other approaches on CIFAR10 at 40\% asymmetric noise rate. Similarly to the symmetric noise cases, we also use  (pseudo-)clean validation sets of sizes 1, 5 and 10 samples per class and show the results of Distill and our method in \cref{tab:assymmetric_noise}~\emph{(table on left)}.
            Although our proposed method does not rely on a manually-labelled validation set, it performs better than Distill, especially with small model architectures (Resnet29) and small validation sets  (1 sample per class).
            Our active selection strategy has slightly lower accuracy with larger clean validation set sizes (larger than or equal 5 random clean samples per classes) on larger model architectures (WideResnet28). This might be caused by the confirmation bias of asymmetric noise in our selected pseudo-clean validation subset and the high capacity of larger models, such as WideResnet28-10, which are more prone to overfit label noise, especially when being trained on a small dataset with only 10 classes.
            We further evaluate the proposed method and show the higher performance of our method compared to other methods, such as FSR and L2R meta-learning methods, in \cref{tab:assymmetric_noise}~\emph{(table on right)}.
            
            
           
            
            
           
            \begin{table}[t]
                \centering
                \caption{Test accuracy (in \%) of our INOLML and previous methods on CIFAR10 with 0.4 asymmetric noise. \emph{(table on left):} comparison with Distill using a validation set $\mathcal{D}^{(v)}$ of sizes 1, 5 and 10 samples per class on Resnet29 and WideResnet28-10, and \emph{(table on right):} comparison with some leading methods. The superscript \textsuperscript{T} indicates the need for clean validation sets.} \label{tab:assymmetric_noise}
                
               
                
                \centering
               
               
                 
                       
                       
                       
                       
                       
                       
                       
                       
                       
                       
                       
                       
                   
                   
                   
                    
                    
                \begin{minipage}[t]{0.48\linewidth}\centering

                \scalebox{0.63}{
                  \begin{tabular}{l c c c}
                        \toprule
                        \bfseries Method & \(\abs{\mathcal{D}^{(v)}}\) & \bfseries Resnet29 & \bfseries WRN28-10  \\
                        \midrule
                        Distill\textsuperscript{T} & \multirow{2}{*}{$1\times C$} & 76.8 \(\pm\) 2.9 & 93.2 \(\pm\) 0.2  \\
                        \textbf{INOLML} & & 86.8 \(\pm\) 0.9 & 93.6 \(\pm\) 0.3 \\
                        \midrule
                        Distill\textsuperscript{T} & \multirow{2}{*}{$5\times C$} & 86.7 \(\pm\) 0.5 & 94.5 \(\pm\) 0.2 \\
                        \textbf{INOLML} & & 89.3 \(\pm\) 0.2 & 94.1 \(\pm\) 0.1 \\
                        \midrule
                        Distill\textsuperscript{T} & \multirow{2}{*}{$10\times C$} & 88.6 \(\pm\) 0.7 & \(\mathbf{94.9} \pm \mathbf{0.1}\)\\
                        \textbf{INOLML} & & \(\mathbf{89.8} \pm \mathbf{0.3}\) & 94.2 \(\pm\) 0.1\\
                        \bottomrule
                    \end{tabular}
                    }
                   
                    \end{minipage}
                   \begin{minipage}[t]{0.48\linewidth}\centering
               
               
               
                \scalebox{0.63}{
                  \begin{tabular}{l c}
                        \toprule
                        \bfseries Method & \bfseries Accuracy \\
                        \midrule
                       
                        GJS\cite{GJS} & 89.7 \(\pm\) 0.4\\
                        F-Correction \cite{assmymetric}  & 83.6 \(\pm\) 0.3\\
                        PENCIL\cite{PENCIL} & 91.2 \(\pm\) 0.0\\
                        DivideMix \cite{li2020dividemix}  & 92.1 \(\pm\) 0.0\\
                        MLNT \cite{li2019learning}  & 92.3 \(\pm\) 0.1 \\
                        \midrule
                        L2R\textsuperscript{T}\cite{L2W} & 90.8 \(\pm\) 0.3\\
                        FSR\cite{FSR} & 93.6 \(\pm\) 0.3\\
                        \rowcolor{gray!30} \textbf{INOLML} & \(\mathbf{94.2} \pm \mathbf{0.1}\) \\
                        \bottomrule
                    \end{tabular}
                    }
                   
               
                \end{minipage}
            \end{table}
           
            
            
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           




       
        \subsection{Imbalanced Learning}
           
            We evaluate our INOLML on the imbalanced (long-tailed) CIFAR datasets following  the same setting as~\cite{FSR}. The prediction accuracy in \cref{tab:table_6} shows that INOLML considerably surpasses all previous meta-learning approaches.
           
            
           
            
            \begin{table}[t]
                \centering
                \caption{Test accuracy (in \%) of our INOLML and other SOTA meta-learning approaches evaluated on the CIFAR imbalanced learning (long-tailed) recognition task. The reported results are from Zhang et al.~\cite{FSR} and Xu et al.~\cite{Famus}.
               
                }
                \label{tab:table_6}
               \scalebox{0.8}{ \begin{tabular}{l c c c c c c}
                    \toprule
                    & \multicolumn{3}{c}{\bfseries CIFAR10} &  \multicolumn{3}{c}{\bfseries CIFAR100}  \\
                    \midrule
                    Imb. ratio & \bfseries 200 & \bfseries 50 & \bfseries 10 & \bfseries 200 & \bfseries 50 & \bfseries 10  \\
                    \cmidrule(lr){2-4} \cmidrule(lr){5-7}
                    Softmax \cite{FSR} &  65.68 & 74.81 & 86.39 &  34.84 & 43.85 & 55.71 \\
                    CB-Focal \cite{FSR} &  65.29 & 76.71 & 86.66 & 32.62 & 44.32 & 55.78 \\
                    CB-Best \cite{FSR} &  68.89 & 79.27 & 87.49  & 36.23 & 45.32 & 57.99 \\
                    \midrule
                    L2R \cite{L2W} &  66.51 & 78.93 & 85.19 &  33.38 & 44.44 & 53.73  \\
                    MWN \cite{MetaWeightNetLA} &  68.91 & 80.06 & 87.84 & 37.91 & 46.74 & 58.46  \\
                   
                   
                    GDW \cite{GDW} &  - & - & 86.8 & - & - & 56.8  \\
                    FaMUS \cite{Famus} &  - & 83.32 & 87.90 & - &  49.93 & 59.03  \\
                    FSR-DF \cite{FSR} &      66.15 & 79.78 & 88.15 & 36.74 & 44.43 & 55.60  \\
                    FSR \cite{FSR} &  67.76 & 79.17 & 87.40 & 35.44 & 42.57 & 55.45  \\
                    \rowcolor{gray!30} \textbf{INOLML} & \(\mathbf{74.91}\) & \(\mathbf{84.43}\) & \(\mathbf{90.81}\)  & \textbf{41.52} & \textbf{51.35} & \textbf{62.07} \\
                    \bottomrule
                \end{tabular}
               }
            \end{table}
            
           
           
           
           
           
           
           
           
                    
           
           
           
           
           
           
                    
           
           
                   
                   
           
           
           
           
           
           
       
        
        \subsection{Imbalanced Noisy-label Learning}
            We evaluate our proposed method in the setting that combines class imbalance and label noise, which has been proposed in~\cite{FSR}. We follow the same experimental configuration by adding 20\% and 40\% symmetric noise to the CIFAR10 imbalanced datasets with different imbalance ratios (10, 50 and 200). 
            The results in \cref{tab:table_7} show that our proposed method outperforms the benchmarks by a large margin. 
            This result is even more remarkable when studying the results with a noise rate of 40\%. 
            For CIFAR100, we show the results for 20\% and 40\% symmetric noise and imbalance ratio 10.  
            We do not show results for larger imbalance ratios because it was not possible to build validation sets with 10 samples per class for the minority classes.
           
            Nevertheless, for the two CIFAR100 problems, our method shows substantially better results than previous SOTA methods.
            Our method can therefore be considered the new SOTA in this imbalanced noisy-label learning benchmark with Resnet32 model.

           
                       
            \begin{table}[t]
                \centering
                \caption{Test accuracy (in \%) of our INOLML and other SOTA methods on CIFAR10 long-tailed recognition mixed with symmetric noise. The reported results are collected from~\cite{FSR} and \cite{ROLT}.
               
                }
                \label{tab:table_7}
                \scalebox{0.8}{
                \begin{tabular}{l l l l l l l l l }
                    \toprule
                    \bfseries Dataset & \multicolumn{6}{c}{\bfseries Cifar10} &\multicolumn{2}{c}{\bfseries Cifar100} \\ 
                    \bfseries Noise ratio& \multicolumn{3}{c}{\bfseries 0.2} &  \multicolumn{3}{c}{\bfseries 0.4} & \bfseries 0.2 & \bfseries 0.4\\
                    \midrule
                    Imb. ratio & \bfseries 10 & \bfseries 50 & \bfseries 200 & \bfseries 10 & \bfseries 50 & \bfseries 200 &\bfseries 10 & \bfseries 10  \\
                    \cmidrule(lr){2-4} \cmidrule(lr){5-7} \cmidrule(lr){8-9}
                    CRUST\cite{CRUST}  &  65.7 & 41.5 & 34.3 &  59.5 & 32.4 & 28.8 & - &- \\
                    LDAM\cite{ LDAM}  &  82.4   & -  & -  &  71.4 & -  &  - &48.1   &36.7 \\
                    LDAM-DRW\cite{LDAM}  &  83.7   & -  & -  &  74.9 & -  & -  &50.4   & 39.4 \\
                    BBN\cite{BBN}  &  80.4 & -  & -  &  70.0 & -  & -  & 47.9 &35.2 \\
                    HAR-DRW\cite{Imbalance_noisy}  &  82.4 & -  & -  &  77.4 & -  & -  & 46.2 & 37.4 \\
                    ROLT-DRW\cite{ROLT}  &  85.5   & -  & -  &  82.0 & -  &  - &52.4   &46.3 \\
                    FSR\cite{FSR}  &  85.7 & 77.4 & 65.5 &  81.6 & 69.8 & 49.5 & - & -   \\
                    \rowcolor{gray!30} \textbf{INOLML} & \(\mathbf{90.1}\) & \(\mathbf{80.1}\) & \(\mathbf{66.6}\) &  \(\mathbf{89.1}\) & \(\mathbf{78.1}\) & \(\mathbf{61.6}\) & \(\mathbf{59.8}\)& \(\mathbf{56.1}\) \\
                    \bottomrule
                \end{tabular}
                }
               
            \end{table}
            
           
                   \subsection{Open-set Noise}
            This type of noise refers to training images that belong to classes falling outside the \(C\) classes in \(\mathcal{D}\). We follow~\cite{lee2019robust}, which forms 3 benchmarks using CIFAR10 contaminated with images from CIFAR100 and ImageNet. We compare with Distill and other meta-learning methods~\cite{GCE,assmymetric,PENCIL,L2W,FSR} in \cref{tab:openset_noise}, and our method shows significant improvements in all benchmarks. In particular, comparing to Distill, our method is 0.5\% to 1\% better. One interesting observation is that our method  outperforms Distill in the open-set noise even though the selected validation set \(\mathcal{D}^{(v)}\) is largely contaminated with noisy samples (up to 40\%) as shown in \cref{fig:probe_acc_open}. 
            This is  in contrast to our previous observation in symmetric and asymmetric noise settings where the more noisy samples in \(\mathcal{D}^{(v)}\), the worse performance of the models trained with our method compared to Distill. 
            Such difference
           
            might be attributed to the out-of-distribution characteristic of open-set noise. As open-set noisy-label datasets contain samples that do not belong to the set of known classes, such samples might help to regularise the learning on mislabelled data, reducing the effect of confirmation bias, resulting in a better performance.
            
            \begin{table}[t]
                \centering
                \caption{Test accuracy (in \%) of our INOLML and previous methods in open-set noise~\cite{OpenSet_noise} using WideResnet28-10 with 10 samples per class for the validation set.
                }
                \label{tab:openset_noise}
                \scalebox{0.8}{
                \begin{tabular}{l l l l}
                    \toprule
                    \bfseries Method & \bfseries ImageNet  & \bfseries CIFAR100  & \bfseries BOTH  \\
                    \midrule
                    RoG~\cite{assmymetric} &  83.4 & 87.1 & 84.4  \\
                    L2R~\cite{L2W} &  81.8 & 81.8 & 85.0  \\
                    Distill~\cite{Distill_noise} &  94.0 & 92.3 & 93.0  \\
                    \rowcolor{gray!30} \textbf{INOLML} &  \(\mathbf{94.5} \pm \mathbf{0.1}\) & \(\mathbf{93.6} \pm \mathbf{0.0}\) & \(\mathbf{93.6} \pm \mathbf{0.1}\) \\
                    \bottomrule
                \end{tabular}
                }
               
            \end{table}
        
           
            \subsection{Real-world Datasets}
            
            
            \cref{tab:sota_results} shows the results of our method and other SOTA approaches on real-world datasets. Except for HAR\cite{Imbalance_noisy} that uses InceptionResnetV2, \cref{tab:sota_results} \emph{(upper table)} shows the performance on WebVision with Resnet50, while \cref{tab:sota_results} \emph{(lower table)} shows results on four different noise ratios evaluated on Red Mini-ImageNet. In general, our method outperforms many SOTA methods on WebVision and is competitive with the best method~\cite{ortego2021multi} on Red Mini-ImageNet. We note that our proposed method is more efficient in terms of memory footprint than most of Co-training based approaches~\cite{li2020dividemix,propmix,Famus} evaluated on Red Mini-ImageNet since we use only a single PreAct Resnet18 model with meta-learning instead of two separate PreAct Resnet18 models.
            

           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
            
            
            
            
            
           
            
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
                   
           
                   
           
           
           
           
            
            
            \begin{table}[t]
                \centering
                \caption{SOTA prediction accuracy (in \%) comparison on real-world datasets. \emph{(upper table):} WebVision mini dataset (50 classes) using Resnet50 evaluated on Webvision and ImageNet test set; and \emph{(lower table):} Red Mini-ImageNet. The results of other methods are reported from Zhang et al.\cite{FSR}, PropMix\cite{propmix} and their own works.}
                \label{tab:sota_results}
                \begin{subtable}[t]{0.5\textwidth}
                \begin{center}
                \scalebox{0.8}{
                    \begin{tabular}{l c  c c c}
                        \toprule
                        \multirow{2}{*}{\bfseries Method} &   \multicolumn{2}{c}{\bfseries WebVision} & \multicolumn{2}{c}{\bfseries ImageNet} \\
                        \cmidrule(lr){2-3} \cmidrule(lr){4-5}
                        & \bfseries top-1 & \bfseries top-5 & \bfseries top-1 & \bfseries top-5  \\
                        \midrule
                        HAR \cite{Imbalance_noisy} &  75.5 & 90.7 &  57.4 & 82.4   \\
                        D2L \cite{ma2018dimensionality}  &  62.7& 84.0  & 57.8 & 81.4  \\
                        Co-teaching \cite{han2018co}  &  63.6 & 85.2 &  61.5 & 84.7   \\
                        Iterative-CV \cite{Chen2019UnderstandingAU}  &  65.2 & 85.3 &  61.6 & 85.0   \\
                        MentorNet \cite{MentorNet} &  63.0 & 81.4 &  63.8 & 85.8  \\
                        CRUST \cite{CRUST}  &  72.4 & 89.6 & 67.4 & 87.8   \\
                        FSR \cite{FSR} &  74.9  & 88.2  & 72.3 & 87.2 \\
                        GJS \cite{GJS}  &  78.0  & 90.6  & 74.4 & 91.2 \\
                        MW-Net \cite{MetaWeightNetLA}  &  74.5 & 88.9  & 72.6 & 88.8 \\
                        
                        \rowcolor{gray!30} \textbf{INOLML}  & \(\mathbf{81.7}\) & \(\mathbf{93.8}\)  & \(\mathbf{78.1}\) & \(\mathbf{92.9}\) \\
                        \bottomrule
                    \end{tabular}
                    }
                    \end{center}
                \end{subtable}
                \hfill
                \begin{subtable}[t]{.5\textwidth}
                \begin{center}
                \scalebox{0.8}{
                    \begin{tabular}{l c c c c}
                        \toprule
                        \multirow{2}{*}{\bfseries Method} & \multicolumn{4}{c}{\bfseries Noise ratio} \\
                        \cmidrule{2-5} 
                        & \bfseries 0.2 & \bfseries 0.4 & \bfseries 0.6 & \bfseries 0.8 \\
                        \cmidrule{1-5}
                        Cross entropy \cite{propmix} & 47.36 & 42.70 & 37.30 & 29.76  \\
                        Mix Up \cite{propmix} & 49.10 &  46.40 & 40.58 & 33.58  \\
                        DivideMix \cite{li2020dividemix} & 50.96 & 46.72 & 43.14 & 34.50  \\
                        MentorMix\cite{jiang2020beyond} & 51.02 & 47.14 & 43.80 & 33.46  \\
                        FaMUS\cite{Famus} & 51.42 & 48.06 & 45.10 & 35.50  \\
                        PropMix\cite{propmix} & 61.24 & 56.22 & 52.84 & 43.42 \\
                        MOIT \cite{ortego2021multi} & 63.14 & \(\mathbf{60.78}\) & - & \(\mathbf{45.88}\) \\ 
                        \rowcolor{gray!30} \textbf{INOLML} & \(\mathbf{63.23}\) & 58.21 & \(\mathbf{53.39}\) & \textbf{45.32}  \\
                       
                        \bottomrule
                    \end{tabular}
                    }
                    \end{center}
                \end{subtable}
               
            \end{table}
            
            
            
           
            
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
           
                   
           
           
           
\section{Algorithm of the proposed method}
\label{sec:algorithm}
    \begin{algorithm}[ht]
        \caption{Training procedure of the proposed INOLML.
        }
        \label{algorithm:training_procedure}
        \begin{algorithmic}[1]
            \Procedure{Training}{$\mathcal{D}$, $\eta$, $T$, $\widetilde{T}$, $T^{(u)}$, $\widetilde{\eta}$, $\kappa,N, M, K, C$}
                \LComment{\(\mathcal{D}\): noisy dataset}
                \LComment{\(\eta\): learning rate per iteration}
                \LComment{\(T\): total number of iterations}
                \LComment{\(\widetilde{T}\): minimum number of iterations before  updating the moving average robust labels}
                \LComment{\(T^{(u)}\): interval between updates}
                \LComment{\(\widetilde{\eta}\): learning rate threshold}
                \LComment{\(\kappa,N, M, K, C\): hyper-parameters}

                \Statex

               
                \State Warmup model $f_{\theta}(.)$ with \(\ell_{\mathrm{CE}}(.)\) from $\mathcal{D}$
                \State $\mathcal{D}^{(c)}=\mathsf{PseudoCleanDetector}\left (\mathcal{D} \right )$ using Equation \ref{eq:PseudoCleanDetector}
                \State Initialise moving average robust label $\{\tilde{\mathbf{y}}_i\}_{i=1}^{|\mathcal{D}^{(c)}|}$ of samples in $\mathcal{D}^{(c)}$
                \State Initialise $\mathcal{D}^{(v)}$ and $\mathcal{D}^{(t)}$ from $\mathcal{D}^{(c)}$  using 
               
                 Equation \ref{eq:select_validation_set}
               
                \State Reinitialize model $f_{\theta}(.)$
                \For{$t = 1$ to $T$}
                    \State Meta-learn to train  \(\theta\), \(\omega\) and \(\lambda\) using Equation \ref{eq:meta_learning_bilevel_optimisation} 
                   
                  
                    \State Update  $\{\tilde{\mathbf{y}}_i\}_{i=1}^{|\mathcal{D}^{(c)}|}$ of samples in $\mathcal{D}^{(c)}$ if  $ t > \widetilde{T}$ and  $\eta_t < \widetilde{\eta}$

                    \State Update $\mathcal{D}^{(c)}=\mathsf{PseudoCleanDetector}\left (\mathcal{D} \right )$ using Equation \ref{eq:PseudoCleanDetector} if  $\eta_t = 0$
                    \If{$T^{(u)} \equiv t \mod T^{(u)}$}
                        \State Update $\mathcal{D}^{(v)}$ and $\mathcal{D}^{(t)}$ from $\mathcal{D}^{(c)}$  using 
               
                 Equation \ref{eq:select_validation_set}
                       
                       
                       
                       
                       
                    \EndIf
                \EndFor
                \State \Return the trained model parameter \(\theta\)
            \EndProcedure
        \end{algorithmic}
    \end{algorithm}
    



\section{Implementation Details}
\label{sec:implementation_details_appendix}
All CIFAR experiments use batches of size 100, which are trained on a single GPU. Similar to the Distill noise model~\cite{Distill_noise}, we use \(p=5, k=20\) for CIFAR experiments, except the ones with the imbalance setting.

For Red Mini-ImageNet experiments, we trained the model on a single  GPU with batches of size 100, with \(p=5, k=10\).

For the WebVision experiment, we use \(p=4, k=8\) with 4 NVIDIA V100 GPU and batches of size 16.  All experiments use \(N=200, K=50, \kappa = 0.9\).

\newpage
\section{Additional Results of Symmetric Noise on CIFAR Datasets}
\label{sec:additional_symmetric_noise_results}
        We provide additional symmetric noise results of our proposed method and the Distill model~\cite{Distill_noise} in Table \ref{tab:table_1}. Note that our method is markedly better than Distill, particularly for the simpler model (RN29) with few samples per class (1 and 5) in the validation set. For the more complex model (WRN) and large validation set (10 samples per class), our method is still better than Distill, except for CIFAR100 at 0.8 symmetric noise rate.

        \begin{table*}[h]
        \centering
        \caption{Test accuracy (in \%) comparison between  our method ('INOLML') and the Distill noise ('DN') on symmetric noise using 1, 5 and 10 samples per class in the validation set on two backbone models: Resnet29 ('RN29') and Wideresnet28-10 ('WRN'). The results of the Distill model with WideResnet28-10 are collected from \cite{Distill_noise}.
        Recall that the Distill needs a clean set, while INOLML works with a pseudo-clean set.}
        
        \label{tab:table_1}
        \scalebox{1.0}{
        \begin{tabular}{l c l | l | l | l | l | l}
            \toprule
            \multirow{3}{*}{\bfseries Method} & \multirow{3}{*}{\shortstack{\bfseries Val. Set\\ \bfseries size}} & \multicolumn{6}{c}{\bfseries Dataset}  \\
           
            & & \multicolumn{3}{c}{\bfseries CIFAR10} & \multicolumn{3}{c}{\bfseries CIFAR100}  \\
            \cmidrule{3-8} 
            & & \multicolumn{1}{c}{\bfseries 0.2} & \multicolumn{1}{c}{\bfseries 0.4} & \multicolumn{1}{c|}{\bfseries 0.8} & \multicolumn{1}{c}{\bfseries 0.2} & \multicolumn{1}{c}{\bfseries 0.4} & \multicolumn{1}{c}{\bfseries 0.8} \\
            \midrule
            DN-RN29  & \multirow{2}{*}{1} & 87.0 \(\pm\) 0.5 & 85.3 \(\pm\) 0.5 & \multicolumn{1}{c|}{FAIL} & 58.9 \(\pm\)  0.5 & 55.8 \(\pm\) 0.7 & \multicolumn{1}{c}{FAIL} \\
            INOLML-RN29  & & 90.3 \(\pm\) 0.2 & 89.1 \(\pm\) 0.5 & 79.1 \(\pm\) 0.3 & 65.9 \(\pm\) 0.2 & 61.5 \(\pm\) 0.2 & 55.1 \(\pm\) 0.1 \\
            \midrule
            DN-RN29  & \multirow{2}{*}{5} & 90.7 \(\pm\) 0.3 &89.0 \(\pm\) 0.3 & 83.5 \(\pm\) 0.2 & 62.6 \(\pm\)  0.4 & 58.8 \(\pm\) 0.5 & 48.5 \(\pm\) 0.5 \\
            INOLML-RN29  & & 90.9 \(\pm\) 0.2 & 90.9 \(\pm\) 0.1&87.4 \(\pm\)  0.2&66.6 \(\pm\)  0.1&65.7 \(\pm\)  0.1&59.0 \(\pm\) 0.5 \\
            \midrule
            DN-RN29  & \multirow{2}{*}{10} & 91.0 \(\pm\) 0.2 & 89.2 \(\pm\) 0.1&87.0 \(\pm\) 0.1&63.7 \(\pm\)  0.2&60.5 \(\pm\) 0.2&57.5  \(\pm\) 0.5 \\
            INOLML-RN29  & & 92.2 \(\pm\) 0.1 & 91.0 \(\pm\) 0.1 & 87.9 \(\pm\)  0.2 & 67.1 \(\pm\) 0.1 & 66.3 \(\pm\)  0.1 & 59.2 \(\pm\) 0.2 \\
            \midrule
            \midrule
            DN-WRN  & \multirow{2}{*}{1} & 95.4 \(\pm\) 0.6 & 94.5 \(\pm\) 1.0 & 87.9 \(\pm\) 5.1 & 77.4 \(\pm\) 0.4 & 75.5 \(\pm\) 1.1 & 62.1 \(\pm\) 1.2 \\
            INOLML-WRN  & & 96.0 \(\pm\) 0.2 &95.9 \(\pm\) 0.2&94.3 \(\pm\) 0.2&81.6 \(\pm\) 0.2&79.5 \(\pm\) 0.2&73.6 \(\pm\) 0.3 \\
            \midrule
            DN-WRN  & \multirow{2}{*}{5} & 96.4 \(\pm\) 0.0 & 95.5 \(\pm\) 0.6 & 91.8  \(\pm\)  3.0 & 80.4 \(\pm\) 0.5 & 79.6  \(\pm\)  0.3 & 73.6 \(\pm\) 1.5 \\
            INOLML-WRN & & 96.4 \(\pm\) 0.1 &96.2 \(\pm\) 0.1 & 94.6 \(\pm\) 0.2 & 82.2 \(\pm\) 0.2&81.5 \(\pm\) 0.2&74.5 \(\pm\)  0.2 \\
            \midrule
            DN-WRN & \multirow{2}{*}{10} & 96.2 \(\pm\) 0.2 &95.9 \(\pm\) 0.2&93.7  \(\pm\)  0.5&81.2 \(\pm\) 0.7&80.2  \(\pm\)  0.3 & \(\mathbf{75.5 \pm 0.2}\) \\
            INOLML-WRN & & \(\mathbf{96.9 \pm 0.1}\) & \(\mathbf{96.6 \pm 0.1}\) & \(\mathbf{95.0 \pm 0.2}\) & \(\mathbf{82.0 \pm 0.2}\) & \(\mathbf{81.3 \pm 0.2}\) & 74.7 \(\pm\) 0.1 \\
            \bottomrule
        \end{tabular}
        }
    \end{table*}

   
   
        
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   

\newpage

        



