\section{Introduction}
\label{sec:intro}
\input{1_intro}

\section{Related Work}
\label{sec:related}
\input{2_related}

\section{Methodology}
\label{sec:method}
\input{3_methodology}

\section{Experiments}
\label{sec:exp}
\input{4_exp}

\section{Conclusion}
\label{sec:conclusion}
\input{5_conclusion}

\begin{acks}
\input{6_ack}
\end{acks}

\bibliographystyle{ACM-Reference-Format}



\subsection{Preliminaries}

\paragraph{Graph Neural Networks.} Consider an attributed graph $\mathcal{G}=\{\mathcal{V}, \mathcal{E}\}$, where $\mathcal{V}$ is the set of nodes and $\mathcal{E}$ is the set of edges. GNNs use the same model framework as follows~\cite{hamilton2017inductive,xu2018powerful}:

\begin{equation}
\label{eq:gnn}
h_i^k=\phi(h_i^{k-1}, \mathcal{F}_{agg}(\{h_j, j\in\mathcal{N}(i)\}))
\end{equation}
\noindent where $\mathcal{F}_{agg}$ represents an aggregation function and $\phi$ represents an update function. The objective of GNNs is to update the embedding of each node by aggregating the information from its neighbor nodes and the connections between them. 

\subsection{GIPA Architecture}

In this part, we present the architecture of GIPA\xspace, which  extracts information from node features as well as edge features in a more general way. The process of GIPA\xspace is shown in~\figurename~\ref{fig:gipa_process}. Consider a node $i$ with feature $h_i$ and its neighbor nodes $j\in\mathcal{N}(i)$ with feature $h_j$. $e_{i, j}$ represents the edge feature between node $i$ and $j$. The problem is how to generate an expected embedding for node $i$ from its own node feature $h_i$, its neighbors' node features $\{h_j\}$ and the related edge features $\{e_{i, j}\}$. 

The GIPA\xspace process mainly consists of three parts: \texttt{attention}, \texttt{propagation} and \texttt{aggregation}. Firstly, GIPA\xspace computes the embedding $\tilde{h}_i$, $\tilde{h}_j$, and $\tilde{e}_{i, j}$ by means of linear projection from the features $h_i$, $h_j$ and $e_{i,j}$, respectively. Secondly, the \texttt{attention} process calculates the multi-head attention weights by using fully-connected layers on $\tilde{h}_i$, $\tilde{h}_j$, and $\tilde{e}_{i, j}$, respectively. Thirdly, the \texttt{propagation} process focuses on propagating information of each neighbor node $j$ by combining $j$'s node embedding $\tilde{h}_j$ with the associated edge embedding $\tilde{e}_{i, j}$. Finally, the \texttt{aggregation} process aggregates all messages from neighbors to update the embedding of $i$. The following subsections introduce the details of each process.

\begin{figure*}
    \centering
    \includegraphics[width=1.0\textwidth]{figures/gipa_process.png}
    \caption{Process of GIPA\xspace, which consists of \textit{attention}, \textit{propagation}, and \textit{aggregation} process.}
    \label{fig:gipa_process}
\end{figure*}

\subsubsection{Attention Process}

Different from the existing attention mechanisms like self-attention or scaled dot-product attention, we use MLP to realize a multi-head attention mechanism.
The \texttt{attention} process of GIPA\xspace can be formulated as follows:

\begin{equation}
\label{eq:att}
\tilde{a}_{i, j} = \mathcal{F}_{att}(\tilde{h}_i, \tilde{h}_j, \tilde{e}_{i, j}|W_{att}) = \text{MLP}([\tilde{h}_i || \tilde{h}_j || \tilde{e}_{i, j}]|W_{att})
\end{equation}

\noindent where the attention function $\mathcal{F}_{att}$ is realized by an \texttt{MLP} with learnable weights $W_{att}$ (without bias). Its input is the concatenation of the node embedding $\tilde{h}_i$ and $\tilde{h}_j$ as well as the edge embedding $\tilde{e}_{i, j}$. As the scale of parameters of \texttt{MLP} is adjustable, this attention mechanism could be more representative than previous ones simply based on dot-product. The final attention weight is calculated by an edge-wise \texttt{softmax} activation function:

\begin{equation}
\label{eq:softmax}
a_{i, j} = \text{softmax}(\tilde{a}_{i, j}) = \frac{\exp(\tilde{a}_{i, j})}{\sum_{k\in N(i)}\exp(\tilde{a}_{i, k})}
\end{equation}

\subsubsection{Propagation Process}

Unlike GAT~\cite{velivckovic2017graph} that only considers the node feature of neighbors, GIPA\xspace incorporates both node and edge features during the \textit{propagation} process:
\begin{equation}
\label{eq:prop}
p_{i, j} = \mathcal{F}_{prop}(\tilde{h}_j, \tilde{e}_{i,j}|W_{prop}) = \text{MLP}([\tilde{h}_j || \tilde{e}_{i, j}]|W_{prop}), 
\end{equation}
\noindent where the propagation function $\mathcal{F}_{prop}$ is also realized by an \text{MLP} with learnable weights $W_{prop}$. Its input is the concatenation of a neighbor node embedding $\tilde{h}_j$ and the related edge embedding $\tilde{e}_{i, j}$. Thus, the $propagation$ is done edge-wise rather than node-wise. 

Combining the results by \texttt{attention} and \texttt{propagation} by element-wise multiplication, GIPA\xspace gets the message $m_{i, j}$ of node $i$ from $j$:
\begin{equation}
\label{eq:msg}
m_{i, j} = a_{i, j} * p_{i, j}
\end{equation}

\subsubsection{Aggregation Process}

For each node $i$, GIPA\xspace repeats previous processes to get messages from its neighbors. The \texttt{aggregation} process first gathers all these messages by a reduce function, summation for example:

\begin{equation}
\label{eq:sum}
m_{i} = \sum_{j\in\mathcal{N}(i)}m_{i, j}
\end{equation}

\noindent Then, a residual connection between the linear projection $\hat{h}_i$ and the message of $m_i$ is added through concatenation: 

\begin{equation}
\label{eq:proj}
\hat{h}_i=W_{proj}\tilde{h}_i
\end{equation}

\begin{equation}
\label{eq:agg}
o_{i} = \mathcal{F}_{agg}(m, \hat{h}_i | W_{agg}) = \text{MLP}([\sum_{j\in\mathcal{N}(i)}m_{i, j} || \hat{h}_i]|W_{agg})
\end{equation}

\noindent where an \texttt{MLP} with learnable weights $W_{agg}$ is applied to get the final output $o_{i}$. 
Finally, we would like to emphasize that the process of GIPA\xspace can be easily extended to multi-layer variants by stacking the process multiple times.
\subsection{Dataset and Settings}

\textbf{Dataset.} In our experiments, the \textit{ogbn-proteins} dataset from OGB~\cite{hu2020open} is used. The dataset is an undirected and weighted graph, containing 132,534 nodes of 8 different species and 79,122,504 edges with 8-dimensional features. The task is a multi-label binary classification problem with 112 classes representing different protein functions. 

\noindent \textbf{Evaluation metric.} The performance is measured by the average ROC-AUC scores. We follow the dataset splitting settings as recommended in OGB and test the performance of 10 different trained models with different random seeds. 

\noindent \textbf{Hyperparameters.} The hyperparamters used in GIPA\xspace and its training process are concluded in \tablename~\ref{tab:hyper}.

\noindent \textbf{Environment.} GIPA\xspace is implemented in Deep Graph Library (DGL)~\cite{wang2019deep} with Pytorch~\cite{paszke2019pytorch} as the backend. The experiments are done in a platform with GeForce TITAN RTX GPU (24GB RAM) and AMD Ryzen Threadripper 3960X CPU (24 Cores).

\begin{table}[t]
\caption{Hyperparameters of GIPA\xspace model.}
\begin{tabular}{lc}
\hline
\textbf{Hyperparameter}      & \textbf{Value} \\ \hline
Node embedding length        & 80             \\ 
Edge embedding length        & 16             \\ 
Number of attention layers   & 2              \\ 
Number of attention heads    & 8              \\ 
Number of propagation layers & 2              \\ 
Number of hidden units       & 80             \\ 
Number of GIPA layers        & 6              \\ 
Edge drop rate               & 0.1            \\ 
Aggregation                  & SUM            \\ 
Activation                   & ReLU           \\
Dropout of node embedding    & 0.1            \\
Dropout of attention         & 0.1            \\ 
Dropout of propagation       & 0.25           \\ 
Dropout of aggregation       & 0.25           \\ 
Dropout of the last FC layer & 0.5            \\ 
Learning rate & 0.01            \\ 
Optimizer & AdamW~\cite{loshchilov2017decoupled}  \\ \hline
\end{tabular}
\label{tab:hyper}
\end{table}
\begin{table}[!h]
\caption{Test and validation performance (ROC-AUC) on \textit{ogbn-proteins} dataset.}
\begin{tabular}{lcc}
\hline
Method              & Test ROC-AUC  & Validation ROC-AUC \\ \hline
MLP                 & $0.7204\pm0.0048$ & $0.7706\pm0.0014$      \\
GCN                 & $0.7251\pm0.0035$ & $0.7921\pm0.0018$      \\
GraphSAGE           & $0.7768\pm0.0020$ & $0.8334\pm0.0013$      \\
DeeperGCN           & $0.8496\pm0.0028$ & $0.8971\pm0.0011$      \\
GAT                 & $0.8682\pm0.0018$ & $0.9194\pm0.0003$      \\
UniMP               & $0.8642\pm0.0008$ & $0.9175\pm0.0006$      \\
UniMP+CrossEdgeFeat & $0.8691\pm0.0018$ & $\mathbf{0.9258\pm0.0009}$      \\
GIPA (ours)         & $\mathbf{0.8700\pm0.0010}$ & $0.9187\pm0.0003$      \\ \hline
\label{tab:performance}
\end{tabular}
\end{table}
\subsection{Performance}
\tablename~\ref{tab:performance} shows the average ROC-AUC and the standard deviation for the test set and the validation set, respectively. The results of the baselines are retrieved from the ogbn-proteins leaderboard\footnotemark[1]. Our GIPA\xspace outperforms all previous methods in the leaderboard and reaches an average test ROC-AUC higher than 0.8700 for the first time.


