Appendix 1: The Proof of (5)
For the right hand side of (4), the first term can be further decomposed as
$$\begin{aligned}&-\,E\left\{ {\left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{\hat{\mathbf{x}}}^{[m]}\right| ^{2}} } \right] \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H} {\hat{\mathbf{x}}}^{[k]}} \right] ^{*}{\hat{\mathbf{x}}}^{[k]}} \right\} \nonumber \\&\quad =-\,E\left\{ {\left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{x}^{[m]}\right| ^{2}} } \right] \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}} \right] ^{*} \mathbf{x}^{[k]}} \right\} \nonumber \\&\qquad +\,subterm1+subterm2 \end{aligned}$$
(11)
where
$$\begin{aligned}&subterm1 \nonumber \\&\quad =-\,E\left\{ {\left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{x}^{[m]}\right| ^{2}} } \right] +\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}\right| ^{2}} \right\} \nonumber \\&\qquad \times \, E\left\{ {\left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right] ^{*}\left( {\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right) } \right\} \nonumber \\&\qquad -\,E\left\{ {\left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right] +\left| {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right| ^{2}} \right\} \nonumber \\&\qquad \times E\left\{ {\left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}} \right] ^{*}\mathbf{x}^{[k]}} \right\} \end{aligned}$$
(12)
$$\begin{aligned}&subterm2=-E\left\{ \left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right] \right. \nonumber \\&\qquad \left. \times \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right] ^{*}\left( {\mathbf{V}^{[k]}\mathbf{n}^{[k]}} \right) \right\} \end{aligned}$$
(13)
The second term can be further decomposed as
$$\begin{aligned}&\left[ {\mathbf{I}_N +(\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}} \right] E\left\{ {\left( \sum _{m=1}^K {|\hat{{y}}_i^{[m]} |^{2}} \right) +|\hat{{y}}_i^{[k]} |^{2}} \right\} \mathbf{w}_i^{[k](n)} \nonumber \\&\quad =E\left\{ {\left( {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{x}^{[m]}\right| ^{2}} } \right) +\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}\right| ^{2}} \right\} \mathbf{w}_i^{[k](n)} \nonumber \\&\qquad +\,subterm3+subterm4 \end{aligned}$$
(14)
where
$$\begin{aligned}&subterm3 \nonumber \\&\quad =E\left\{ {\left( {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{V}^{[m]}\mathbf{n}^{[m]}\right| ^{2}} } \right) +\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}\right| ^{2}} \right\} \mathbf{w}_i^{[k](n)} \nonumber \\&\qquad +\,(\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}E\left\{ \left( {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}\mathbf{x}^{[m]}\right| ^{2}} } \right) \right. \nonumber \\&\qquad \left. +\,\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}\right| ^{2} \right\} \mathbf{w}_i^{[k](n)} \end{aligned}$$
(15)
$$\begin{aligned}&subterm4 \nonumber \\&\quad =(\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}E\left\{ \left( {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}\mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right) \right. \nonumber \\&\qquad \left. +\,\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}\right| ^{2} \right\} \mathbf{w}_i^{[k](n)} \end{aligned}$$
(16)
The third term can be further simplified as
$$\begin{aligned} \begin{array}{l} E\left\{ {{\hat{\mathbf{x}}}^{[k]}\left( {{\hat{\mathbf{x}}}^{[k]}} \right) ^{T}} \right\} \cdot E\left\{ {\left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{\hat{\mathbf{x}}}^{[k]}} \right] ^{2*}} \right\} \mathbf{w}_i^{[k](n)*} \\ \quad =E\left\{ {\mathbf{x}^{[k]}\left( {\mathbf{x}^{[k]}} \right) ^{T}} \right\} E\left\{ {\left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}} \right] ^{2*}} \right\} \mathbf{w}_i^{[k](n)*} \\ \end{array} \end{aligned}$$
(17)
Thus, (4) can be further rewritten as
$$\begin{aligned} \mathbf{w}_i^{[k](n+1)}= & {} -\,E\left\{ {\left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{x}^{[m]}\right| ^{2}} } \right] \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}} \right] ^{*}{} \mathbf{x}^{[k]}} \right\} \nonumber \\&+\,E\left\{ {\left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{x}^{[m]}\right| ^{2}} } \right] +\left| {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}} \right| ^{2}} \right\} \mathbf{w}_i^{[k](n)} \nonumber \\&+\,E\left\{ {\mathbf{x}^{[k]}\left( {\mathbf{x}^{[k]}} \right) ^{T}} \right\} E\left\{ {\left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{x}^{[k]}} \right] ^{2*}} \right\} \mathbf{w}_i^{[k](n)*} \nonumber \\&+\,f\left( {{\begin{array}{cccc} {\mathbf{w}_i^{[1](n)} }&{} {\mathbf{w}_i^{[2](n)} }&{} \cdots &{} {\mathbf{w}_i^{[K](n)} } \\ \end{array} }} \right) \end{aligned}$$
(18)
where
$$\begin{aligned}&f\left( {{\begin{array}{cccc} {\mathbf{w}_i^{[1](n)} }&{} {\mathbf{w}_i^{[2](n)} }&{} \cdots &{} {\mathbf{w}_i^{[K](n)} } \\ \end{array} }} \right) \\&\quad =-\,E\left\{ {\left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right] \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right] ^{*}\left( {\mathbf{V}^{[k]}\mathbf{n}^{[k]}} \right) } \right\} \\&\qquad +\,(\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}E\left\{ \left( {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right) \right. \nonumber \\&\qquad \left. +\,\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}\mathbf{n}^{[k]}\right| ^{2} \right\} \mathbf{w}_i^{[k](n)} \end{aligned}$$
Appendix 2: The Proof of (8)
Due to the noise of different groups are uncorrelated, it can be obtained that
$$\begin{aligned}&-\,E\left\{ {\left[ {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right] \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right] ^{*}\left( {\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right) } \right\} \nonumber \\&\quad =-\,E\left\{ {\left[ {\left| {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right| ^{2}} \right] \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right] ^{*}\left( {\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right) } \right\} \nonumber \\&\qquad -\,E\left\{ {\left[ {\sum _{m\ne k} {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right] } \right\} \cdot (\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}{} \mathbf{w}_i^{[k](n)} \end{aligned}$$
(19)
and
$$\begin{aligned}&(\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}E\left\{ {\left( {\sum _{m=1}^K {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}\mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right) +\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}\right| ^{2}} \right\} \mathbf{w}_i^{[k](n)} \nonumber \\&\quad =(\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}E\left\{ {2\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}\mathbf{n}^{[k]}\right| ^{2}} \right\} \mathbf{w}_i^{[k](n)} \nonumber \\&\qquad +\,(\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}E\left\{ {\left( {\sum _{m\ne k} {\left| \left( {\mathbf{w}_i^{[m](n)} } \right) ^{H}{} \mathbf{V}^{[m]}{} \mathbf{n}^{[m]}\right| ^{2}} } \right) } \right\} \mathbf{w}_i^{[k](n)} \end{aligned}$$
(20)
Thus, (8) can be obtained if
$$\begin{aligned}&-\,E\left\{ {\left[ {\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}\right| ^{2}} \right] \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right] ^{*}\left( {\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right) } \right\} \nonumber \\&\quad +\,2(\sigma ^{[k]})^{2}{} \mathbf{V}^{[k]}(\mathbf{V}^{[k]})^{H}E\left\{ {\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}\right| ^{2}} \right\} \mathbf{w}_i^{[k](n)} =\mathbf{0} \end{aligned}$$
(21)
is true. Next, we will show that the Eq. (21) is true.
Due to the linear combination of independent Gaussian random variables is also Gaussian, each component of \({\bar{\mathbf{n}}}^{[k]}\triangleq \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}\) is a Gaussian variable.
Then, \(\hbox {kurt}\left( {\bar{{n}}_i^{[k]} } \right) =E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{4}} \right\} -2\left( {E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{2}} \right\} } \right) ^{2}-E\left\{ {\bar{{n}}_i^{[k]2} } \right\} E\left\{ {\bar{{n}}_i^{[k]*2} } \right\} =0\ (i=1,2,\cdots N)\).
Considering the effect of \(E\left( {{\bar{\mathbf{n}}}^{[k]}{\bar{\mathbf{n}}}^{{[k]}{T}}} \right) =\mathbf{V}^{[k]}E\left( {\mathbf{n}^{[k]}\mathbf{n}^{{[k]}{T}}} \right) \mathbf{V}^{{[k]}{T}}=\mathbf{0}\), we can get \(\hbox {kurt}\left( {\bar{{n}}_i^{[k]} } \right) =E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{4}} \right\} -2\left( {E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{2}} \right\} } \right) ^{2}=0,\) i.e., \(E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{4}} \right\} =2\left( {E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{2}} \right\} } \right) ^{2}\).
Furthermore,
$$\begin{aligned} E\left( {{\bar{\mathbf{n}}}^{[k]}{\bar{\mathbf{n}}}^{{[k]}{H}}} \right)= & {} \mathbf{V}^{[k]}E\left( {\mathbf{n}^{[k]}{} \mathbf{n}^{{[k]}{H}}} \right) \mathbf{V}^{{[k]}{H}} \\= & {} \sigma ^{{[k]}{2}}{\varvec{\Lambda } }^{{[k]}{-1}}\triangleq diag\left( {{\begin{array}{cccc} {E\left\{ {\left| {\bar{{n}}_1^{[k]} } \right| ^{2}} \right\} }&{} {E\left\{ {\left| {\bar{{n}}_2^{[k]} } \right| ^{2}} \right\} }&{} \cdots &{} {E\left\{ {\left| {\bar{{n}}_N^{[k]} } \right| ^{2}} \right\} } \\ \end{array} }} \right) \\ \end{aligned}$$
Thus, for the left hand side of (21), the first term can be rewritten explicitly as
$$\begin{aligned}&-\,E\left\{ {\left[ {\left| \left( {\mathbf{w}_i^{[k](n)} } \right) ^{H} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}\right| ^{2}} \right] \left[ {\left( {\mathbf{w}_i^{[k](n)} } \right) ^{H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right] ^{*}\left( {\mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right) } \right\} \nonumber \\&\quad =-\,E\left\{ {\left| {\mathbf{w}^{[k](n)H} {\bar{\mathbf{n}}}^{[k]}} \right| ^{2}\cdot (\mathbf{w}^{[k](n)H} {\bar{\mathbf{n}}}^{[k]})^{*}\cdot {\bar{\mathbf{n}}}^{[k]}} \right\} \nonumber \\&\quad =-\,E\left\{ {\sum _{a=1}^N {\left( {w_a^{[k](n)*} \bar{{n}}_a^{[k]} } \right) } \cdot \sum _{b=1}^N {\left( {w_b^{[k](n)*} \bar{{n}}_b^{[k]} } \right) ^{*}} \cdot \sum _{c=1}^N {\left( {w_c^{[k](n)*} \bar{{n}}_c^{[k]} } \right) ^{*}} \cdot {\bar{\mathbf{n}}}^{[k]}} \right\} \end{aligned}$$
(22)
The \(i^{th}\) element can be further expressed as
$$\begin{aligned}&-\left| {w_i^{[k](n)} } \right| ^{2}w_i^{[k](n)} E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{4}} \right\} +2w_i^{[k](n)} E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{2}} \right\} E\left\{ {\sum _{l\ne i} {\left| {w_l^{[k](n)} } \right| ^{2}\left| {\bar{{n}}_l^{[k]} } \right| ^{2}} } \right\} \nonumber \\&\quad =-2w_i^{[k](n)} E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{2}} \right\} E\left\{ {\sum _{l=1}^N {\left| {w_l^{[k](n)} } \right| ^{2}\left| {\bar{{n}}_l^{[k]} } \right| ^{2}} } \right\} \end{aligned}$$
(23)
where \(i=1,2\cdots N\).
The second term can be rewritten explicitly as
$$\begin{aligned}&2\sigma ^{[k]2}{} \mathbf{V}^{[k]}{} \mathbf{V}^{[k]H}E\left\{ {\left| {\mathbf{w}^{[k](n)H}{} \mathbf{V}^{[k]}{} \mathbf{n}^{[k]}} \right| ^{2}} \right\} \mathbf{w}^{[k](n)} \nonumber \\&\quad =2\sigma ^{[k]2}{} \mathbf{V}^{[k]}{} \mathbf{V}^{[k]H}E\left\{ {\left| {\mathbf{w}^{[k](n)H}{\bar{\mathbf{n}}}^{[k]}} \right| ^{2}} \right\} \mathbf{w}^{[k](n)} \nonumber \\&\quad =2\sigma ^{[k]2}{} \mathbf{V}^{[k]}{} \mathbf{V}^{[k]H}E\left( {\sum _{l=1}^N {\left| {w_l^{[k](n)} } \right| ^{2}\left| {\bar{{n}}_l^{[k]} } \right| ^{2}} } \right) \mathbf{w}^{[k](n)} \end{aligned}$$
(24)
The ith element can be further expressed as
$$\begin{aligned} 2w_i^{[k](n)} E\left\{ {\left| {\bar{{n}}_i^{[k]} } \right| ^{2}} \right\} E\left\{ {\sum _{l=1}^N {\left| {w_l^{[k](n)} } \right| ^{2}\left| {\bar{{n}}_l^{[k]} } \right| ^{2}} } \right\} \end{aligned}$$
(25)
where \(i=1,2\cdots N\).
Therefore, (21) is true.
Thus, (8) is obtained based on (19), (20) and (21).
