\[ \begin{aligned} \operatorname{Var}\mathopen{}\left(\tilde{a}\cdot \tilde{X}\right)\mathclose{} &= \operatorname{Var}\mathopen{}\left(\sum_{i=1}^na_i X_i\right)\mathclose{} \\ &= \tilde{a}^{\top} \operatorname{Var}\mathopen{}\left(\tilde{X}\right)\mathclose{} \tilde{a} \\ &= \sum_{i=1}^n\sum_{j=1}^n a_i a_j \operatorname{Cov}\mathopen{}\left(X_i,X_j\right)\mathclose{} \end{aligned} \]
\[\operatorname{E}\mathopen{}\left[Y\right]\mathclose{} = \operatorname{E}\mathopen{}\left[\operatorname{E}\mathopen{}\left[Y \mid X\right]\mathclose{}\right]\mathclose{}\]
\[\operatorname{E}\mathopen{}\left[Y \mid Z\right]\mathclose{} = \operatorname{E}\mathopen{}\left[\operatorname{E}\mathopen{}\left[Y \mid X,Z\right]\mathclose{} \mid Z\right]\mathclose{}\]
\[\operatorname{Var}\mathopen{}\left(Y\right)\mathclose{} = \operatorname{E}\mathopen{}\left[\operatorname{Var}\mathopen{}\left(Y \mid X\right)\mathclose{}\right]\mathclose{} + \operatorname{Var}\mathopen{}\left(\operatorname{E}\mathopen{}\left[Y \mid X\right]\mathclose{}\right)\mathclose{}\]
\[\operatorname{Cov}\mathopen{}\left(Y,Z\right)\mathclose{} = \operatorname{E}\mathopen{}\left[\operatorname{Cov}\mathopen{}\left(Y,Z \mid X\right)\mathclose{}\right]\mathclose{} + \operatorname{Cov}\mathopen{}\left(\operatorname{E}\mathopen{}\left[Y \mid X\right]\mathclose{}, \operatorname{E}\mathopen{}\left[Z \mid X\right]\mathclose{}\right)\mathclose{}\]
\[\mathcal{L}(\theta) \stackrel{\text{def}}{=}\operatorname{p}(\tilde{X}= \tilde{x}| \Theta = \theta)\]
\[\ell\stackrel{\text{def}}{=}\operatorname{log}\mathopen{}\left\{\mathcal{L}(\tilde{x}|\theta)\right\}\mathclose{}\]
\[\ell'\stackrel{\text{def}}{=}\frac{\partial}{\partial \theta} \ell(\tilde{x}|\theta)\]
\[\ell''\stackrel{\text{def}}{=}\frac{\partial}{\partial \tilde{\theta}}\frac{\partial}{\partial \tilde{\theta}^{\top}} \ell(\tilde{x}| \tilde{\theta})\]
\[\ell_{ij}''= \frac{\partial}{\partial \theta_i}\frac{\partial}{\partial \theta_j} \ell(\tilde{X}= \tilde{x}| \tilde{\theta})\]
\[I\stackrel{\text{def}}{=}-\ell''(\tilde{x}|\tilde{\theta})\] \[\mathcal{I}\stackrel{\text{def}}{=}\operatorname{E}\mathopen{}\left[I(\tilde{x}|\theta)\right]\mathclose{}\]
\[\hat\theta_{ML}\ \dot \sim\ \operatorname{N}\mathopen{}\left(\theta,\mathopen{}\left[\mathcal{I}(\tilde{\theta})\right]\mathclose{}^{-1}\right)\mathclose{}\]
For one parameter \(\theta_k\):
\[ \text{CI}_{1-\alpha}(\theta_k) = \left[ \hat\theta_k \pm z_{1 - \frac{\alpha}{2}} \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat\theta_k\right)\mathclose{} \right] \]
\[ Z_k \stackrel{\text{def}}{=} \frac{\hat\theta_k-\theta_{k,0}}{\mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat\theta_k\right)\mathclose{}} \ \dot{\sim} \ \operatorname{N}\mathopen{}\left(0,1\right)\mathclose{} \qquad \text{under }H_0:\theta_k=\theta_{k,0} \qquad z_k\text{ = observed }Z_k \]
\[ \begin{aligned} p\text{-value} &= 2\Pr \mathopen{}\left(\mathopen{}\left|Z\right|\mathclose{}\ge \mathopen{}\left|z_k\right|\mathclose{}\right)\mathclose{},\quad Z\sim \operatorname{N}\mathopen{}\left(0,1\right)\mathclose{} \\ &= 2\mathopen{}\left(1-\Phi\mathopen{}\left(\mathopen{}\left|z_k\right|\mathclose{}\right)\mathclose{}\right)\mathclose{} \end{aligned} \]
\[ t_k \stackrel{\text{def}}{=} \frac{\hat \beta_k-\beta_{k,0}}{\mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat \beta_k\right)\mathclose{}} \ \dot{\sim} \ t_{n-p} \qquad \text{under }H_0:\beta_k=\beta_{k,0} \qquad t_k^{\text{obs}}\text{ = observed }t_k \]
\[ \text{CI}_{1-\alpha}(\beta_k) = \left[ \hat \beta_k \pm t_{n-p}\mathopen{}\left(1-\frac{\alpha}{2}\right)\mathclose{} \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat \beta_k\right)\mathclose{} \right] \]
Let \(T_{n-p}\sim t_{n-p}\).
\[ \begin{aligned} p\text{-value} &= 2\Pr \mathopen{}\left(\mathopen{}\left|T_{n-p}\right|\mathclose{}\ge \mathopen{}\left|t_k^{\text{obs}}\right|\mathclose{}\right)\mathclose{} \\ &= 2\mathopen{}\left(1-F_{t_{n-p}}\mathopen{}\left(\mathopen{}\left|t_k^{\text{obs}}\right|\mathclose{}\right)\mathclose{}\right)\mathclose{} \end{aligned} \]
\[ \text{CI}_{1-\alpha}\mathopen{}\left(\mu\mathopen{}\left({\tilde{x}^*}\right)\mathclose{}\right)\mathclose{} = \left[ \hat{\mu}\mathopen{}\left({\tilde{x}^*}\right)\mathclose{} \pm t_{n-p}\mathopen{}\left(1-\frac{\alpha}{2}\right)\mathclose{} \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat{\mu}\mathopen{}\left({\tilde{x}^*}\right)\mathclose{}\right)\mathclose{} \right] \]
\(Y^*\) denotes a new observation (not in the training data), with corresponding covariate pattern \({\tilde{x}^*}\). Let \(\hat{Y}^* \stackrel{\text{def}}{=}\hat{\mu}\mathopen{}\left({\tilde{x}^*}\right)\mathclose{}\).
\[ \text{PI}_{1-\alpha}\mathopen{}\left(Y^*|{\tilde{x}^*}\right)\mathclose{} = \left[ \hat{Y}^* \pm t_{n-p}\mathopen{}\left(1-\frac{\alpha}{2}\right)\mathclose{} \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(Y^*-\hat{Y}^*\right)\mathclose{} \right] \]
\[ \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(Y^*-\hat{Y}^*\right)\mathclose{} = \hat \sigma\sqrt{1 + {({\tilde{x}^*})}^{\top}(\mathbf{X}'\mathbf{X})^{-1}{\tilde{x}^*}} \]
\[ \operatorname{Var}\mathopen{}\left(Y^* - \hat{Y}^*\right)\mathclose{} = \sigma^2\mathopen{}\left(1 + {({\tilde{x}^*})}^{\top}(\mathbf{X}'\mathbf{X})^{-1}{\tilde{x}^*}\right)\mathclose{} \]
Let \(\hat{\Sigma} \stackrel{\text{def}}{=}\mathop{\widehat{\operatorname{Var}}}\nolimits\mathopen{}\left(\hat \beta\right)\mathclose{} = \hat \sigma^2(\mathbf{X}'\mathbf{X})^{-1}\).
\[ \mathop{\widehat{\operatorname{Var}}}\nolimits\mathopen{}\left(\hat{\mu}\mathopen{}\left(\tilde{x}\right)\mathclose{}\right)\mathclose{} = {\tilde{x}}^{\top}\hat{\Sigma}\tilde{x} \]
Let \(\Delta\mu\mathopen{}\left(\tilde{x}, {\tilde{x}^*}\right)\mathclose{} = \mu\mathopen{}\left(\tilde{x}\right)\mathclose{} - \mu\mathopen{}\left({\tilde{x}^*}\right)\mathclose{}\), and let \(\Delta\tilde{x}= \tilde{x}- {\tilde{x}^*}\); then:
\[ \mathop{\widehat{\operatorname{Var}}}\nolimits\mathopen{}\left(\widehat{\Delta\mu}\mathopen{}\left(\tilde{x}, {\tilde{x}^*}\right)\mathclose{}\right)\mathclose{} = {\Delta\tilde{x}}^{\top}\hat{\Sigma}\Delta\tilde{x} \]
Generalized linear models have three components:
The outcome distribution family: \(\operatorname{p}(Y|\mu(\tilde{x}))\)
The link function: \(g(\mu(\tilde{x})) = \eta(\tilde{x})\)
The linear component: \(\eta(\tilde{x}) = \tilde{x}\cdot \beta\)
\[\theta_{\omega}(\tilde{x},{\tilde{x}^*}) = \operatorname{exp}\mathopen{}\left\{(\Delta\tilde{x}) \cdot \tilde{\beta}\right\}\mathclose{}\]
\[\hat\theta=\frac{ad}{bc}\]
These association-measure results are covered in Measures of Association for Binary Outcomes.
Odds (def) \[\omega\stackrel{\text{def}}{=}\frac{\Pr(A)}{\Pr(\neg A)}\]
Conditional odds (def) \[\omega(A|B) \stackrel{\text{def}}{=}\frac{\Pr(A|B)}{\Pr(\neg A|B)}\]
Odds function (def) \[\operatorname{odds}\mathopen{}\left\{\pi\right\}\mathclose{} \stackrel{\text{def}}{=}\frac{\pi}{1-\pi}\]
Probability to odds (thm, cor) \[\omega= \frac{\pi}{1-\pi} = \operatorname{odds}\mathopen{}\left\{\pi\right\}\mathclose{}\]
Simplified odds expressions; odds of a non-event (thm, cor) \[\operatorname{odds}\mathopen{}\left\{\pi\right\}\mathclose{} = \frac{1}{\pi^{-1}-1} = \mathopen{}\left(\pi^{-1}-1\right)^{-1}\mathclose{}, \qquad \omega(\neg A) = \frac{1-\pi}{\pi} = \pi^{-1}-1\]
Odds ratio (def) \[\theta(\omega_1, \omega_2) \stackrel{\text{def}}{=}\frac{\omega_1}{\omega_2}\]
OR as ratio of probability ratios (thm) \[\theta(\omega_1, \omega_2) = \frac{\omega_1}{\omega_2} = \frac{\mathopen{}\left(\frac{\pi_1}{1-\pi_1}\right)\mathclose{}}{\mathopen{}\left(\frac{\pi_2}{1-\pi_2}\right)\mathclose{}}\]
Odds ratios are reversible, marginally and conditionally (thm, thm) \[\theta_{\omega}(A|B) = \theta_{\omega}(B|A), \qquad \theta_{\omega}(A|B,C) = \theta_{\omega}(B|A,C)\]
Odds to probability (thm) \[\pi = \frac{\omega}{1+\omega}\]
Inverse-odds function (def; recovers probability by cor) \[\operatorname{invodds}\mathopen{}\left\{\omega\right\}\mathclose{} \stackrel{\text{def}}{=}\frac{\omega}{1 + \omega}, \qquad \pi= \operatorname{invodds}\mathopen{}\left\{\omega\right\}\mathclose{}\]
Simplified inverse-odds (lem) \[\operatorname{invodds}\mathopen{}\left\{\omega\right\}\mathclose{} = \frac{1}{1+\omega^{-1}} = (1+\omega^{-1})^{-1}\]
One minus inverse-odds, and its complement (lem, cor) \[1 - \pi= \frac{1}{1+\omega}, \qquad 1+\omega= \frac{1}{1-\pi}\]
Log-odds, and log-odds from probability (def, thm) \[\eta\stackrel{\text{def}}{=}\operatorname{log}\mathopen{}\left\{\omega\right\}\mathclose{}, \qquad \eta= \operatorname{log}\mathopen{}\left\{\frac{\pi}{1-\pi}\right\}\mathclose{}\]
Logit function, and expanded form (def, thm) \[\operatorname{logit}(\pi) \stackrel{\text{def}}{=}\operatorname{log}\mathopen{}\left\{\operatorname{odds}\mathopen{}\left\{\pi\right\}\mathclose{}\right\}\mathclose{} = \operatorname{log}\mathopen{}\left\{\frac{\pi}{1-\pi}\right\}\mathclose{}\]
Log-odds equals logit (cor) \[\eta= \operatorname{logit}\mathopen{}\left\{\pi\right\}\mathclose{}\]
Odds and probability from log-odds (lem, thm) \[\omega= \operatorname{exp}\mathopen{}\left\{\eta\right\}\mathclose{}, \qquad \pi= \frac{\operatorname{exp}\mathopen{}\left\{\eta\right\}\mathclose{}}{1+\operatorname{exp}\mathopen{}\left\{\eta\right\}\mathclose{}}\]
Expit / inverse-logit function, and equivalent expressions (def, thm) \[\operatorname{expit}(\eta) \stackrel{\text{def}}{=}\operatorname{invodds}\mathopen{}\left\{\operatorname{exp}\mathopen{}\left\{\eta\right\}\mathclose{}\right\}\mathclose{} = \frac{\operatorname{exp}\mathopen{}\left\{\eta\right\}\mathclose{}}{1+\operatorname{exp}\mathopen{}\left\{\eta\right\}\mathclose{}} = (1 + \operatorname{exp}\mathopen{}\left\{-\eta\right\}\mathclose{})^{-1}\]
Probability as expit (thm) \[\pi= \operatorname{expit}\mathopen{}\left\{\eta\right\}\mathclose{} \tag{1}\]
Logit and expit are inverses (thm) \[\operatorname{logit}\mathopen{}\left\{\operatorname{expit}\mathopen{}\left\{\eta\right\}\mathclose{}\right\}\mathclose{} = \eta\qquad \operatorname{expit}\mathopen{}\left\{\operatorname{logit}\mathopen{}\left\{\pi\right\}\mathclose{}\right\}\mathclose{} = \pi\]
\[ \underbrace{\pi}_{\Pr\mathopen{}\left(Y=1\right)\mathclose{}} \overbrace{ \underbrace{ \underset{ \xleftarrow[\frac{\omega}{1+\omega}]{} } { \xrightarrow{\frac{\pi}{1-\pi}} } \underbrace{\omega}_{\operatorname{odds}\mathopen{}\left\{Y=1\right\}\mathclose{}} \underset{ \xleftarrow[\operatorname{exp}\mathopen{}\left\{\eta\right\}\mathclose{}]{} } { \xrightarrow{\operatorname{log}\mathopen{}\left\{\omega\right\}\mathclose{}} } }_{\operatorname{expit}\mathopen{}\left\{\eta\right\}\mathclose{}} }^{\operatorname{logit}\mathopen{}\left\{\pi\right\}\mathclose{}} \underbrace{\eta}_{\eta\mathopen{}\left(Y=1\right)\mathclose{}} \]
Odds minus probability (thm) \[\omega- \pi = \frac{\pi^2}{1-\pi}, \quad \text{where } \omega= \frac{\pi}{1-\pi}\]
OR as RR times a correction factor (thm) \[\theta(\omega_1, \omega_2) = \rho(\pi_1, \pi_2) \cdot \frac{1-\pi_2}{1-\pi_1}\] When \(\pi_1\) and \(\pi_2\) are both small, the correction factor is close to 1, so the odds ratio is close to the risk ratio.
\[ \begin{array}{r|ccccc} & \pi& \omega& \eta& \tilde{x}& \tilde{\beta}\\ \hline \pi & 1 & \mathopen{}\left(1+\omega\right)^2\mathclose{} & \dfrac{\mathopen{}\left(1+\omega\right)^2\mathclose{}}{\omega} & \text{undef} & \text{undef} \\[10pt] \omega & \mathopen{}\left(1-\pi\right)^2\mathclose{} & 1 & \dfrac{1}{\omega} & \text{undef} & \text{undef} \\[10pt] \eta & \pi(1-\pi) & \omega & 1 & \text{undef} & \text{undef} \\[10pt] \tilde{x} & \underbrace{\tilde{\beta}\pi(1-\pi)}_{p \times 1} & \underbrace{\tilde{\beta}\omega}_{p \times 1} & \underbrace{\tilde{\beta}}_{p \times 1} & \underbrace{\mathbb{I}}_{p \times p} & \mathbf{0}_{p \times p} \\[10pt] \tilde{\beta} & \underbrace{\tilde{x}\pi(1-\pi)}_{p \times 1} & \underbrace{\tilde{x}\omega}_{p \times 1} & \underbrace{\tilde{x}}_{p \times 1} & \mathbf{0}_{p \times p} & \underbrace{\mathbb{I}}_{p \times p} \\ \end{array} \]
Column labels indicate the numerators of the derivatives; row labels indicate the denominators (dimensions and details in tbl).
Derivative of expit (cor) \[\operatorname{expit}'\mathopen{}\left\{\eta\right\}\mathclose{} = \operatorname{expit}\mathopen{}\left\{\eta\right\}\mathclose{}(1 - \operatorname{expit}\mathopen{}\left\{\eta\right\}\mathclose{})\]
Logistic regression (def) \[Y_i|\tilde{X}_i \ \sim_{\perp\!\!\!\perp}\ \operatorname{Ber}(\pi(\tilde{X}_i)), \qquad \operatorname{logit}\mathopen{}\left\{\pi(\tilde{x})\right\}\mathclose{} = {\tilde{x}}^{\top}\tilde{\beta}\]
Throughout this summary, \(\tilde{\beta}= (\beta_0, \beta_1, \ldots, \beta_p)\) is the \((p+1) \times 1\) coefficient vector (including the intercept), \(\mathbf{X}\) is the \(n \times (p+1)\) design matrix whose \(i^{\text{th}}\) row is \({\tilde{x}_i}^{\top}\), and \(\tilde{\varepsilon}\) is the \(n \times 1\) error vector with components \(\varepsilon_i \stackrel{\text{def}}{=}y_i - \pi_i\).
Log-likelihood component (lem) \[\ell_i(\pi_i) = y_i \eta_i - \operatorname{log}\mathopen{}\left\{1+\omega_i\right\}\mathclose{}\]
Score function as sum of components (lem, thm) \[\tilde{\ell'}(\tilde{\beta}) = \sum_{i=1}^n\tilde{\ell'_i}(\tilde{\beta}), \qquad \ell_i'(\tilde{\beta}) = \tilde{x}_i \varepsilon_i\]
Score function (thm) \[\tilde{\ell'}(\tilde{\beta}) = \sum_{i=1}^n\tilde{x}_i\varepsilon_i = \underbrace{{\mathbf{X}}^{\top}}_{(p+1) \times n}\underbrace{\tilde{\varepsilon}}_{n \times 1}\]
Estimating equation (eq) The MLE \(\hat{\tilde{\beta}}\) solves the score equation: \[\underbrace{\tilde{\ell'}(\hat{\tilde{\beta}})}_{(p+1) \times 1} = \sum_{i=1}^n\underbrace{\tilde{x}_i}_{(p+1) \times 1}\underbrace{\mathopen{}\left(y_i - \operatorname{expit}\mathopen{}\left\{{\tilde{x}_i}^{\top}\hat{\tilde{\beta}}\right\}\mathclose{}\right)\mathclose{}}_{1 \times 1} = \mathbf{0}_{(p+1) \times 1}\]
This estimating equation has no closed-form solution in general; \(\hat{\tilde{\beta}}\) must be computed by numerical iteration.
Hessian (eq) \[\underbrace{\ell''(\tilde{\beta})}_{(p+1) \times (p+1)} = -\underbrace{{\mathbf{X}}^{\top}}_{(p+1) \times n} \underbrace{\mathbf{D}}_{n \times n} \underbrace{\mathbf{X}}_{n \times (p+1)}, \qquad \mathbf{D} \stackrel{\text{def}}{=}\text{diag}(\operatorname{Var}\mathopen{}\left(Y_i|\tilde{X}_i=\tilde{x}_i\right)\mathclose{}) = \text{diag}(\pi_i(1-\pi_i))\]
Concavity (rem): the Hessian is negative semi-definite for every \(\tilde{\beta}\) (negative definite when \(\mathbf{X}\) has full column rank), so the log-likelihood is concave and every solution of the score equation is a global maximum.
Newton-Raphson update (Newton-Raphson method) \[\underbrace{{\widehat{\tilde{\beta}}}^*}_{(p+1) \times 1} \leftarrow \underbrace{{\widehat{\tilde{\beta}}}^*}_{(p+1) \times 1} - \underbrace{\mathopen{}\left(\ell''\mathopen{}\left(\tilde{y}; {\widehat{\tilde{\beta}}}^*\right)\mathclose{}\right)^{-1}\mathclose{}}_{(p+1) \times (p+1)} \underbrace{\ell'\mathopen{}\left(\tilde{y}; {\widehat{\tilde{\beta}}}^*\right)\mathclose{}}_{(p+1) \times 1}\]
One-sample MLE for odds (thm) \[\hat{\omega}= \frac{x}{n-x}\]
Wald test statistic for \(H_0: \beta_k = \beta_{k,0}\) (CLT for MLEs) \[z_k = \frac{\hat \beta_k - \beta_{k,0}}{\mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat \beta_k\right)\mathclose{}}, \qquad z_k \ \dot{\sim} \ N(0,1) \text{ under } H_0\]
95% confidence intervals for \(\beta_k\) and \(e^{\beta_k}\) (MLE invariance) \[\hat \beta_k \pm 1.96 \cdot \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat \beta_k\right)\mathclose{}, \qquad e^{\beta_k} \in \mathopen{}\left(e^{\hat \beta_k - 1.96 \cdot \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat \beta_k\right)\mathclose{}},\; e^{\hat \beta_k + 1.96 \cdot \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat \beta_k\right)\mathclose{}}\right)\mathclose{}\]
For \(k \neq 0\), \(e^{\beta_k}\) is an odds ratio; for \(k = 0\), \(e^{\beta_0}\) is the baseline odds.
General OR formula (lem) \[\theta_{\omega}(\tilde{x},{\tilde{x}^*}) = \operatorname{exp}\mathopen{}\left\{\eta(\tilde{x}) - \eta({\tilde{x}^*})\right\}\mathclose{}\]
Difference in log-odds; OR in terms of \(\Delta\eta\) (def, cor) \[\Delta\eta\stackrel{\text{def}}{=}\eta(\tilde{x}) - \eta({\tilde{x}^*}), \qquad \theta_{\omega}(\tilde{x},{\tilde{x}^*}) = \operatorname{exp}\mathopen{}\left\{\Delta\eta\right\}\mathclose{}\]
\(\Delta\eta\) from covariates (lem) \[\Delta\eta= (\tilde{x}- {\tilde{x}^*})\cdot\tilde{\beta}\]
Difference in covariate patterns; \(\Delta\eta\) from \(\Delta\tilde{x}\) (def, cor) \[\Delta\tilde{x}\stackrel{\text{def}}{=}\tilde{x}- {\tilde{x}^*}, \qquad \Delta\eta= \Delta\tilde{x}\cdot \tilde{\beta}\]
OR in terms of \(\Delta\tilde{x}\) (thm) \[\theta_{\omega}(\tilde{x},{\tilde{x}^*}) = \operatorname{exp}\mathopen{}\left\{(\Delta\tilde{x}) \cdot \tilde{\beta}\right\}\mathclose{}\]
Log OR equals \(\Delta\eta\) (cor) \[\operatorname{log}\mathopen{}\left\{\theta_{\omega}(\tilde{x},{\tilde{x}^*})\right\}\mathclose{} = \Delta\eta\]
Estimated SE of log-odds (thm) \[\mathop{\widehat{\operatorname{Var}}}\nolimits\mathopen{}\left(\hat\eta(\tilde{x})\right)\mathclose{} = \underbrace{{\tilde{x}}^{\top}}_{1 \times (p+1)}\underbrace{\hat{\Sigma}}_{(p+1) \times (p+1)}\underbrace{\tilde{x}}_{(p+1) \times 1}, \qquad \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\hat\eta(\tilde{x})\right)\mathclose{} = \sqrt{{\tilde{x}}^{\top}\hat{\Sigma}\tilde{x}}\]
Estimated SE of \(\Delta\widehat{\eta}\) (thm) \[\mathop{\widehat{\operatorname{Var}}}\nolimits\mathopen{}\left(\Delta{\hat\eta}\right)\mathclose{} = \underbrace{{\Delta\tilde{x}}^{\top}}_{1 \times (p+1)}\underbrace{\hat{\Sigma}}_{(p+1) \times (p+1)}\underbrace{(\Delta\tilde{x})}_{(p+1) \times 1}, \qquad \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\Delta{\hat\eta}\right)\mathclose{} = \sqrt{{\Delta\tilde{x}}^{\top}\hat{\Sigma}(\Delta\tilde{x})}\]
CI for an odds ratio (eq; details in sec): exponentiate the endpoints of the CI for \(\Delta\eta\): \[\theta_{\omega}(\tilde{x},{\tilde{x}^*}) \in (e^L, e^R), \qquad (L, R) = \widehat{\Delta\eta}\mp 1.96 \cdot \mathop{\widehat{\operatorname{SE}}}\nolimits\mathopen{}\left(\widehat{\Delta\eta}\right)\mathclose{}\]
Disease and exposure odds ratios (def, def) \[\theta_{\omega}(D|E) \stackrel{\text{def}}{=}\frac{\omega(D|E)}{\omega(D|\neg E)}, \qquad \theta_{\omega}(E|D) \stackrel{\text{def}}{=}\frac{\omega(E|D)}{\omega(E|\neg D)}\]
Case-control studies (section): outcome-stratified sampling breaks estimation of risks, risk ratios, and the components of the disease odds ratio (def), but the exposure odds ratio (def) is estimable, and it equals the disease odds ratio by reversibility (thm).
2×2 table shortcut (table; cell counts \(a, b\) = exposed events and non-events, \(c, d\) = unexposed events and non-events): \[\hat{\omega}(Event|Exposed) = \frac{a}{b}, \qquad \hat{\omega}(Event|\neg Exposed) = \frac{c}{d}, \qquad \theta_{\omega}(Exposed, \neg Exposed) = \frac{ad}{bc}\]
Details in sec; here \(q\) is the number of distinct covariate patterns, \(p\) is the number of \(\beta\) parameters, and \(\ell_{\text{full}}\) is the saturated-model log-likelihood.
Deviance test (def) \[D \stackrel{\text{def}}{=}2(\ell_{\text{full}} - \ell(\hat \beta; \mathbf x)), \qquad D \ \dot{\sim} \ \chi^2(q - p)\]
Hosmer-Lemeshow test (def; \(g\) groups by predicted probability) \[X^2 \stackrel{\text{def}}{=}\sum_{c=1}^g \frac{(o_c - e_c)^2}{e_c} \ \dot{\sim} \ \chi^2(g-2)\]
Pearson chi-squared GOF test (sum of squared Pearson residuals, def) \[X^2 = \sum_{k=1}^q X_k^2 \ \dot{\sim} \ \chi^2(q-p)\]
AIC and BIC [lower is better] (AIC (Akaike 1974); BIC (Schwarz 1978)) \[\text{AIC} = -2 \ell(\hat\theta) + 2p, \qquad \text{BIC} = -2 \ell(\hat\theta) + p \operatorname{log}\mathopen{}\left\{n\right\}\mathclose{}\]
Likelihood ratio test (def; for nested models with \(p_0 < p_1\) parameters — here \(\ell_1\) and \(\ell_0\) are the two fitted models’ log-likelihoods, not the saturated model’s) \[\Lambda \stackrel{\text{def}}{=}2(\ell_1(\hat\theta_1) - \ell_0(\hat\theta_0)), \qquad \Lambda \ \dot{\sim} \ \chi^2(p_1 - p_0) \text{ under } H_0\] by Wilks’ theorem.
Residuals are computed per covariate pattern \(k\) (grouped data); \(h_k\) denotes the leverage of pattern \(k\) (def).
Response residual (def) \[e_k \stackrel{\text{def}}{=}\bar y_k - \hat{\pi}(x_k)\]
Pearson residual (and standardized version) (def, def) \[X_k \stackrel{\text{def}}{=}\frac{\bar y_k - \hat\pi_k}{\sqrt{\hat \pi_k (1-\hat\pi_k)/n_k}}, \qquad r_{P_k} \stackrel{\text{def}}{=}\frac{X_k}{\sqrt{1-h_k}}\]
Deviance residual (and standardized version) (def) \[d_k \stackrel{\text{def}}{=}\operatorname{sign}(y_k - n_k \hat \pi_k)\left\{\sqrt{2[\ell_{\text{full}}(x_k) - \ell(\hat \beta; x_k)]}\right\}, \qquad r_{D_k} \stackrel{\text{def}}{=}\frac{d_k}{\sqrt{1-h_k}}\]
Cook’s distance (def) \[D_k \stackrel{\text{def}}{=}\frac{(r_{P_k})^2}{p} \cdot \frac{h_k}{1 - h_k}\]
Leverage (def): \(h_k\) is the \(k\)-th diagonal element of the weighted hat matrix (with \(\mathbf{X}\) here the \(q \times p\) design matrix of covariate patterns): \[\underbrace{\mathbf{H}}_{q \times q} \stackrel{\text{def}}{=} \underbrace{\mathbf{W}^{1/2}}_{q \times q} \underbrace{\mathbf{X}}_{q \times p} \mathopen{}\left(\underbrace{{\mathbf{X}}^{\top}\,\mathbf{W}\,\mathbf{X}}_{p \times p}\right)\mathclose{}^{-1} \underbrace{{\mathbf{X}}^{\top}}_{p \times q} \underbrace{\mathbf{W}^{1/2}}_{q \times q}, \qquad \underbrace{\mathbf{W}}_{q \times q} \stackrel{\text{def}}{=}\text{diag}\mathopen{}\left(n_k \, \hat\pi_k \, (1 - \hat\pi_k)\right)\mathclose{}\]
Risk difference (def) \[\delta(\pi_1,\pi_2) \stackrel{\text{def}}{=}\pi_1 - \pi_2\]
Risk ratio (def) \[\rho(\pi_1,\pi_2) \stackrel{\text{def}}{=}\frac{\pi_1}{\pi_2}\]
Relative risk difference (def; equals RR minus 1 by thm) \[\xi(\pi_1,\pi_2) \stackrel{\text{def}}{=}\frac{\delta(\pi_1,\pi_2)}{\pi_2} = \rho(\pi_1,\pi_2) - 1\]
Conditional predicted risk (def) \[\pi(a, z) \stackrel{\text{def}}{=}\Pr(Y = 1 \mid A = a, Z = z)\]
Conditional RD and RR (def, def) \[\text{RD}(z) \stackrel{\text{def}}{=}\pi(1, z) - \pi(0, z), \qquad \text{RR}(z) \stackrel{\text{def}}{=}\frac{\pi(1, z)}{\pi(0, z)}\]
Observed marginal risk (def; standardization form by thm) \[\pi(a) \stackrel{\text{def}}{=}\Pr(Y = 1 \mid A = a) = \operatorname{E}\mathopen{}\left[\pi(a, Z) \mid A = a\right]\mathclose{}\]
Potential mean and causal marginal risk (def, def; identification form by thm) \[\mu_a \stackrel{\text{def}}{=}\operatorname{E}\mathopen{}\left[Y^a\right]\mathclose{}, \qquad \pi_a \stackrel{\text{def}}{=}\Pr(Y^a = 1) = \operatorname{E}\mathopen{}\left[\pi(a, Z)\right]\mathclose{}\]
No confounding: observed = causal (cor; the two estimands are contrasted in rem) \[Z \perp\!\!\!\perp A \implies \pi(a) = \pi_a\]
G-computation estimator (def; consistent for \(\pi_a\) by thm) \[\hat\pi_a \stackrel{\text{def}}{=}\frac{1}{n}\sum_{i=1}^n\hat\pi_{a,i}, \qquad \hat\pi_{a,i} \stackrel{\text{def}}{=}\hat\pi(a, z_i)\]
Predictive margins (def; full-sample form def is consistent for \(\pi_a\) by cor) \[\widehat{\text{PM}}(a \mid S) \stackrel{\text{def}}{=}\frac{1}{|S|} \sum_{i \in S} \hat\pi_{a,i}, \qquad \widehat{\text{PM}}(a) \stackrel{\text{def}}{=}\frac{1}{n}\sum_{i=1}^n\hat\pi_{a,i} = \hat\pi_a\]
Exposed-subgroup predictive margin (def; consistent for \(\pi(a)\) by thm; equals \(\bar Y_a\) exactly by lem) \[\hat\pi_a(a) \stackrel{\text{def}}{=}\frac{1}{n_a} \sum_{i : A_i = a} \hat\pi_{a,i} = \bar Y_a\]
Observed marginal RD and RR (def, def) \[\text{OMRD} \stackrel{\text{def}}{=}\pi(1) - \pi(0), \qquad \text{OMRR} \stackrel{\text{def}}{=}\frac{\pi(1)}{\pi(0)}\]
Causal marginal RD and RR (def, def) \[\text{CMRD} \stackrel{\text{def}}{=}\pi_1 - \pi_0, \qquad \text{CMRR} \stackrel{\text{def}}{=}\frac{\pi_1}{\pi_0}\]
Bootstrap SE and CI for marginal contrasts (def; see Bootstrap Confidence Intervals and exm): resample the data with replacement \(B\) times; refit the logistic model and recompute the marginal contrast on each resample; the bootstrap SE is the standard deviation of the \(B\) estimates, and a 95% percentile CI uses their 2.5th and 97.5th percentiles.
Collapsibility (def): an estimand \(\theta\) is collapsible with respect to \(Z\) if \[\theta \stackrel{\text{def}}{=}\operatorname{E}\mathopen{}\left[\theta(Z)\right]\mathclose{}\] Risk differences are collapsible; risk ratios and odds ratios are not collapsible in general (thm; counterexample in exm).
| Name | Symbols | Definition |
|---|---|---|
| Probability density function (PDF) | \(\operatorname{f}(t), \operatorname{p}(t)\) | \(\operatorname{p}(T=t)\) |
| Cumulative distribution function (CDF) | \(\operatorname{F}(t), \operatorname{P}(t)\) | \(\operatorname{P}(T\leq t)\) |
| Survival function | \(\operatorname{S}(t), \bar{\operatorname{F}}(t)\) | \(\operatorname{P}(T > t)\) |
| Hazard function | \(\lambda(t), \operatorname{h}(t)\) | \(\operatorname{p}(T=t|T\ge t)\) |
| Cumulative hazard function | \(\Lambda(t), \operatorname{H}(t)\) | \(\int_{u=-\infty}^t {\lambda}(u)du\) |
| Log-hazard function | \(\eta(t)\) | \(\operatorname{log}\mathopen{}\left\{{\lambda}(t)\right\}\mathclose{}\) |
\[ \operatorname{f}(t) \xleftarrow[\operatorname{S}(t){\lambda}(t)]{-S'(t)} \operatorname{S}(t) \xleftarrow[]{\operatorname{exp}\mathopen{}\left\{-{\Lambda}(t)\right\}\mathclose{}} {\Lambda}(t) \xleftarrow[]{\int_{u=0}^t {\lambda}(u)du} {\lambda}(t) \xleftarrow[]{\operatorname{exp}\mathopen{}\left\{\eta(t)\right\}\mathclose{}} \eta(t) \]
\[ \operatorname{f}(t) \xrightarrow[\int_{u=t}^\infty \operatorname{f}(u)du]{\operatorname{f}(t)/{\lambda}(t)} \operatorname{S}(t) \xrightarrow[-\log{\operatorname{S}(t)}]{} {\Lambda}(t) \xrightarrow[{\Lambda}'(t)]{} {\lambda}(t) \xrightarrow[\operatorname{log}\mathopen{}\left\{{\lambda}(t)\right\}\mathclose{}]{} \eta(t) \]
\[ \begin{aligned} \operatorname{p}(Y=y,D=d) &= [\operatorname{f}_T(y)]^{d} [\operatorname{S}_T(y)]^{1-d} \\ &= [{\lambda}_T(y)]^{d} [\operatorname{S}_T(y)] \end{aligned} \]
\[\hat{{\lambda}}_i = \frac{d_i}{r_i}\]
\[\hat{\kappa}_i = 1 - \hat{{\lambda}}_i = \frac{r_i - d_i}{r_i}\]
\[\mathop{\hat{\operatorname{S}}}\nolimits_{\text{KM}}\mathopen{}\left(t\right)\mathclose{} \stackrel{\text{def}}{=}\prod_{\mathopen{}\left\{i:\ t_i \le t\right\}\mathclose{}} \hat{\kappa}_i\]
\[\hat{{\Lambda}}_{NA}(t) \stackrel{\text{def}}{=}\sum_{\mathopen{}\left\{i:\ t_i \le t\right\}\mathclose{}}\hat{{\lambda}}_i\]
| Formula | Description |
|---|---|
| \({\lambda}(t\mid\tilde{x}) = {\lambda}_0(t)\cdot\theta_{{\lambda}}(\tilde{x})\) | Proportional hazards assumption |
| \({\Lambda}(t\mid\tilde{x}) = {\Lambda}_0(t)\cdot\theta_{{\lambda}}(\tilde{x})\) | Cumulative hazard factorization |
| \(\eta(t\mid\tilde{x}) = \eta_0(t) + \Delta\eta(\tilde{x})\) | Log-hazard decomposition |
| \(\Delta\eta(\tilde{x}) = \tilde{x} \cdot \tilde{\beta} = \beta_1 x_1 + \cdots + \beta_p x_p\) | Linear predictor |
| \(\theta_{{\lambda}}(\tilde{x}) = \operatorname{exp}\mathopen{}\left\{\Delta\eta(\tilde{x})\right\}\mathclose{}\) | Hazard multiplier |
| \(\theta_{{\lambda}}(t| \tilde{x}: {\tilde{x}^*}) \stackrel{\text{def}}{=}\frac{{\lambda}(t|\tilde{x})}{{\lambda}(t|{\tilde{x}^*})}\) | Hazard ratio definition |
| \(\theta_{{\lambda}}(t\mid\tilde{x}:{\tilde{x}^*}) = \operatorname{exp}\mathopen{}\left\{\Delta\eta(\tilde{x}) - \Delta\eta({\tilde{x}^*})\right\}\mathclose{} = \operatorname{exp}\mathopen{}\left\{(\tilde{x}-{\tilde{x}^*}) \cdot \tilde{\beta}\right\}\mathclose{}\) | Hazard ratio formula |
Definition 1 (Cox PH partial likelihood) \[ \begin{aligned} \mathcal{L}^*_i(\tilde{\beta}) &\stackrel{\text{def}}{=}\frac{{\color{red}{\theta_{{\lambda}}(\tilde{x}_{(i)})}}}{\sum_{k \in R(t_i)} {\color{red}{\theta_{{\lambda}}(\tilde{x}_k)}}} \\ \mathcal{L}^*(\tilde{\beta}) &\stackrel{\text{def}}{=} \prod_{\mathopen{}\left\{i:\ d_i = 1\right\}\mathclose{}} \mathcal{L}^*_i(\tilde{\beta}) \end{aligned} \]
where, for the distinct event times \(t_1 < \cdots < t_K\) of the data set:
Definition 2 (Breslow estimator of the baseline cumulative hazard) \[\hat {\Lambda}_0(t) \stackrel{\text{def}}{=} \sum_{t_i \le t} \frac{d_i}{\sum_{k\in R(t_i)} \theta_{{\lambda}}(\tilde{x}_k)}\]
where, for the distinct event times \(t_1 < \cdots < t_K\) of the data set:
Example 1 (Breslow estimator with two event times) Suppose there are two distinct event times \(t_1 = 2\) and \(t_2 = 4\) (no ties, \(d_1 = d_2 = 1\)), with risk-set-weighted totals \(\sum_{k \in R(2)} \theta_{{\lambda}}(\tilde{x}_k) = 10\) and \(\sum_{k \in R(4)} \theta_{{\lambda}}(\tilde{x}_k) = 6\). Then
\[ \hat{\Lambda}_0(4) = \frac{1}{10} + \frac{1}{6} \approx 0.267 . \]