\documentclass[11pt]{amsproc}
\textwidth=6in
\calclayout
\usepackage{amsmath,amssymb}
\usepackage{mathrsfs}
\def\argmin{\mathop{\arg\min}\limits}
\newcommand\norm[1]{\|#1\|}
\def\idot{i_{\bullet}}
\def\kdot{k_{\bullet}}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{corollary}{Corollary}[section]
\newtheorem{lemma}{Lemma}[section]
\newtheorem{proposition}{Proposition}[section]
\theoremstyle{definition}               %%Change Theoremstyle
\newtheorem{assumption}{Assumption}[section]
\newtheorem{defn}{Definition}[section]
\newtheorem{example}{Example}[section]
\newtheorem{remark}{Remark}[section]
%%%%%%%%
\newcommand{\thmref}[1]{Theorem~\ref{thm.#1}}
\newcommand{\propref}[1]{Proposition~\ref{prop.#1}}
\newcommand{\lmref}[1]{Lemma~\ref{lm.#1}}
\newcommand{\corref}[1]{Corollary~\ref{cor.#1}}
\newcommand{\defnref}[1]{Definition~\ref{defn.#1}}
\newcommand{\convref}[1]{Convention~\ref{defn.#1}}
\newcommand{\remref}[1]{Remark~\ref{rem.#1}}
\newcommand{\exref}[1]{Example~\ref{ex.#1}}
\newcommand{\assref}[1]{Assumption~{#1}}
\newcommand{\Ref}[1]{(\ref{#1})}
%%%%%%%%%%%%
\newcommand{\Thm}[1]{\begin{theorem}\label{thm.#1}}
\newcommand{\Cor}[1]{\begin{corollary}\label{cor.#1}}
\newcommand{\Lm}[1]{\begin{lemma}\label{lm.#1}}
\newcommand{\Prop}[1]{\begin{proposition}\label{prop.#1}}
\newcommand{\ass}[1]{\begin{assumption}\label{ass.#1}}
\newcommand{\Defn}[1]{\begin{defn}\label{defn.#1}}
\newcommand{\Ex}[1]{\begin{example}\label{ex.#1}}
\newcommand{\Conv}[1]{\begin{conv}\label{conv.#1}}
\newcommand{\Rem}[1]{\begin{remark}\label{rem.#1}}
\newcommand{\prob}[1]{\begin{problem}\label{prob.#1}}
%%%%%%%%%%%%%
\newcommand{\new}{\setlength{\parindent}{0pt}\vspace{5.0pt}
  \setlength{\hangindent}{1.6pc}\setlength{\hangafter}{1}}
%Q.E.D.
\def\sqr#1#2{{\quad\vbox{\hrule height.#2pt
        \hbox{\vrule width.#2pt height#1pt \kern#1pt
             \vrule width.#2pt}
                     \hrule height.#2pt}}}
\def\qed{{\sqr53}}

\numberwithin{equation}{section}
\renewcommand{\theequation}{\arabic{section}.\arabic{equation}}

\def\und{\quad\textrm{and}\quad}
\def\R{\mathbb{R}}
\def\1{\mathbf{1}}
\def\trace{\mathrm{trace}}
\def\al{\underline{a}}
\def\au{\overline{a}}
\def\ve{\varepsilon}
\def\hve{\hat\ve}
\def\vt{\vartheta}
\def\hvt{\hat\vt}
\def\hs{\hat \sigma}
\def\si{\sum_{i=1}^n}
\def\avi{\frac1n \si}
\def\avl{\dfrac{1}{n}\sum_{l=1}^n}
\def\savj{\frac{1}{\sqrt{n}} \sj}
\def\wav{\frac{1}{nc_n^q} \sj w\Big(\frac{X_j-x}{c_n}\Big)}
\def\maxj{\max_{1\leq j \leq n}}
\def\opn{o_p(n^{-1/2})}
\def\op{o_p(1)}
\def\D{\mathcal{D}}
\def\hD{\hat \Delta_n}
\def\cube{\mathscr{C}}
\def\H{\mathcal{H}}
\def\half{\frac12\,}
\def\Var{\mathrm{Var}}
\def\hb{\hat \beta_m}
\def\bm{\beta_m}
\def\hM{\hat M_m}
\def\ahat{\hat a}
\def\chat{\hat c}
\def\rhat{\hat \rho}
%\newcommand{\tel}[1]{\begin{equation}\tag{#1}}
\newcommand{\bel}[1]{\begin{equation}\label{#1}}
\newcommand{\ee}{\end{equation}}
\def\bea{\[\begin{aligned}}
\def\ea{\end{aligned}\]}
\def\emp{\mathbb{F}}
\def\hemp{\hat{\mathbb{F}}}
\def\temp{\tilde{\mathbb{F}}}

\def\da{\dot a}
\def\hW{\hat W}
\def\hG{\hat G}
\def\hH{\hat H}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}
\title[Conditionally constrained models]{Simple efficient estimators \\
in conditionally constrained models}
\author{Ursula U. M\"uller, Anton Schick
 and Wolfgang Wefelmeyer}

\address{Ursula U. M\"uller,
Department of Statistics, 
Texas A\&M University,
College Station, TX 77843-3143, USA}
\address{Anton Schick,
Department of Mathematical Sciences,
Binghamton University,
Binghamton, NY 13902-6000, USA}
\address{Wolfgang Wefelmeyer,
Mathematical Institute,
University of Cologne,
Weyertal 86--90,
50931 Cologne, Germany}


\begin{abstract}
We consider regression models described by parametrized
conditional constraints. The parameter can be estimated efficiently
by a weighted least squares estimator 
and a generalized method of moments estimator,
both with weights that are estimated conditional expectations.
More recently, a one-step improvement of an initial
estimator and an empirical maximum likelihood estimator have been studied. 
Here we suggest a fifth efficient estimator. 
It is also a weighted least squares estimator,
but now with weights involving only estimated \emph{unconditional} 
expectations, together with an orthonormal basis 
for the square-integrable functions of the covariates.
Using a similar approach, we also construct 
efficient estimators for expectations of functions  
of covariates and response.
\end{abstract}


\maketitle

\emph{Key words:} Nonlinear regression model, quasi-likelihood model.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\section{Introduction} 
\label{sec1}


Suppose we observe independent copies $(X_1,Y_1),\dots,(X_n,Y_n)$
of a pair of random variables $(X,Y)$ which follows a regression model 
given by conditional constraints 
\[
E(a_\vt(X,Y)|X)=0.
\]
Here the covariate $X$ is $d$-dimensional, the parameter $\vt$ 
is $p$-dimensional, and $a_\vt$ is a $k$-dimensional vector of functions.
A simple example is the nonlinear regression model $Y=r_\vt(X)+\ve$
with $E(\ve|X)=0$, for which $a_\vt(X,Y)=Y-r_\vt(X)$. 
The following four estimators for $\vt$ 
have been studied extensively.

1. (\emph{Weighted least squares}.)
For a $k \times p$-matrix $W_\vt(X)$ of weight functions, 
the \emph{weighted least squares estimator} (WLSE) $\hvt_W$ of $\vt$
is a solution of the estimating equation
\bel{w}
\si W_\vt^\top(X_i)a_\vt(X_i,Y_i) = 0.
\ee
It admits a stochastic approximation of the following form.
Let $P$, $Q_X$, and $R$ denote the joint distribution of $(X,Y)$,
the conditional distribution of $Y$ given $X$, and the marginal 
distribution of $X$. For a function $b(X,Y)$ we use the abbreviations 
$Pb=E[b(X,Y)]$, $Q_X b=E(b(X,Y)|X)$. Similarly, for a function $c(X)$
we write $Rc=E[c(X)]$. Under appropriate assumptions [ref\dots], 
$\vt_W$ is \emph{asymptotically linear} with \emph{influence function}
$-(P[W_\vt^\top \da_\vt])^{-1}W_\vt^\top(X) a_\vt(X,Y)$.
This means that 
\bel{wls}
\hvt = \vt - (P[W_\vt^\top \da_\vt])^{-1}\avi W_\vt^\top(X_i) a_\vt(X_i,Y_i)
+ \op.
\ee
By the central limit theorem, its asymptotic covariance matrix is
\[
(P[W_\vt^\top \da_\vt])^{-1}P[W_\vt^\top a_\vt a_\vt^\top W_\vt]
(P[\da_\vt^\top W_\vt])^{-1}.
\]
The asymptotic covariance matrix is minimized by the choice
\[
W_\vt(X) = W_\vt^*(X) = (Q_Xa_\vt a_\vt^\top)^{-1}Q_X\da_\vt.
\]
This optimal weight function $W_\vt^*(X)$ depends 
on the unknown conditional distribution $Q_X$ of $Y$ given $X$
and must be replaced by an estimated version $\hW_\vt^*(X)$, 
for example by replacing
the conditional expectations $Q_Xa_\vt a_\vt^\top$ and $Q_X\da_\vt$ 
by Nadaraya--Watson kernel estimators. This does not change the asymptotic
covariance matrix. The resulting optimally weighted least squares estimator
$\hvt_{\hW^*}$ is asymptotically linear with influence function
$J^{-1}\ell(X,Y)$,
where
\bea
\ell(X,Y) &= - Q_X\da_\vt^\top (Q_Xa_\vt a_\vt^\top)^{-1} a_\vt(X,Y), \\
J &= P[\ell\ell^\top] = R[Q\da_\vt^\top(Qa_\vt a_\vt^\top)^{-1}Q\da_\vt].
\ea
This means that $\hvt_{\hW^*}$ has the stochastic approximation
\bel{eff}
\hvt = \vt + J^{-1}\avi\ell(X_i,Y_i) + \op.
\ee
This is the characterization of estimators that are regular and efficient
for $\vt$ in the sense of the H\'ajek--Le Cam convolution theorem [ref \dots].

2. (\emph{Generalized method of moments}.)
Let $M_n$ be a sequence of random $p\times p$-matrices converging
in probability to a deterministic matrix $M$.
Let $W_\vt(X)$ be a $k \times p$-matrix of weight functions.
The \emph{generalized method of moments estimator} (GMME) $\hvt_{MW}$
minimizes
\[
\si a_\vt^\top(X_i,Y_i) W_\vt(X_i,Y_i) M_n
\si W_\vt^\top(X_i,Y_i) a_\vt(X_i,Y_i).
\]
This estimator is also asymptotically linear
and has the same influence function as the weighted least squares 
estimator $\hvt_W$. In particular, the influence function does not
depend on the matrix $M$. The optimal weights are again $W_\vt^*$.

3. (\emph{One-step improvement}.)
If $\tilde\vt$ is a consistent estimator of $\vt$,
another efficient estimator of $\vt$ is obtained by the \emph{one-step}
(or \emph {Newton--Raphson}) improvement
\[
\hvt = \tilde\vt + \hat J^{-1}\avi\hat\ell(X_i,Y_i),
\] 
where $\hat J$ and $\hat\ell(X,Y)$ are appropriate estimators 
of $J$ and $\ell(X,Y)$. [ref\dots]

4. (\emph{Maximum empirical likelihood estimator.})
Let $G$ denote the distribution function of the covariate $X$.
Our model $Q_X a_\vt=0$ is characterized by infinitely 
many \emph{unconditional} constraints
\[
E[f(X)a_\vt(X,Y)] = 0 \quad\mbox{for all}\quad f\in L_2(G).
\]
Choose an orthonormal basis (ONB) $\phi_0$, $\phi_1$, \dots of $L_2(\lambda^d)$,
where $\lambda^d$ is the uniform distribution on the unit cube $[0,1]^d$.
Assume that $G$ is continuous. Then $G(X)$ is uniformly distributed
on $[0,1]$, and $\phi_0(G(X))$, $\phi_1(G(X))$, \dots is an ONB 
of $L_2(G)$. Hence the model $Q_X a_\vt=0$ is also characterized by 
the countably many unconditional constraints
\[
E[\phi_j(G(X))a_\vt(X,Y)] = 0, \quad j=0,1,\dots
\]
Let $\Pi_n=\{(\pi_1,\dots,\pi_n):\pi_i\geq 0,\si\pi_i=1\}$. 
Denote the empirical distribution function based on the covariates
by $\hat G(t)=(1/n)\si\1[X_i\leq t]$. Set
\[
\hat R_n(\vt) = \sup\Big\{\prod_{i=1}^n n\pi_i:
(\pi_1,\dots,\pi_n)\in\Pi_n,\si\phi_j(\hG(X_i))a_\vt(X_i,Y_i)=0
\Big\}.
\]
The \emph{maximum empirical likelihood estimator} (MELE) $\hvt_{MELE}$
is a maximizer of $R_n(\vt)$. It can be shown that it has the stochastic
approximation \Ref{eff}; hence it is also efficient.

\bigskip
We now suggest a fifth efficient estimator for $\vt$.
It is related to the MELE but simpler.
For each $j=0,1,\dots$ and each $k\times p$-matrix $M_{j\vt}$
with real entries we obtain an estimating equation
\[
M_{j\vt}^\top\si\phi_j(\hG(X_i))a_\vt(X_i,Y_i)=0. 
\]
We combine $m$ such estimating equations,
\[
\sum_{j=1}^m M_{j\vt}^\top\si\phi_j(\hG(X_i))a_\vt(X_i,Y_i)=0. 
\] 
Setting $W_\vt(X)=\phi(\hG(X))M_\vt$ in \Ref{w}, we see that, for fixed $m$,
the solution $\hvt$ of this estimating equation is asymptotically linear
with influence function [\dots] and asymptotic covariance matrix [\dots].
The asymptotic covariance matrix is minimized for the choice
\[
M = M^* = \dots.
\]
This choice depends on the underlying distribution $P$
and must be estimated, for example by plugging in empirical estimators [\dots].
This does not change the asymptotic covariance matrix.
In Section 2 we set $m=m_n\to\infty$ and show that the solution
of the corresponding estimating equation [\dots] is asymptotically linear 
with influence function $g=J^{-1}\ell$, and hence asymptotically efficient
for $\vt$.

We can use a variant of this approach to estimate expectations
of the form $E[h(X,Y)]$ for a known function $h\in L_2(P)$.
Here we could also have used an \emph{empirical maximum likelihood estimator}
\[
\avi \hat\pi_{\hvt i}h(X_i,Y_i),
\] 
where $\hvt$ is an efficient estimator of $\hvt$, for example
one of the five estimators above, and where $\hat\pi_{\vt i}$
are the maximizers of $\prod n\pi_i$ over $(\pi_1,\dots,\pi_n)\in\Pi_n$
with the constraints 
\[
\si\pi_i\phi_j(\hG(X_i))a_\vt(X_i,Y_i) = 0 
\quad\mbox{for}\quad j=1,\dots,m_n.
\]
We suggest another, asymptotically equivalent, estimator for $E[h(X,Y)]$
that is constructed analogously to our ONB estimator for $\vt$.

It is obtained from an initial estimator by adding a correction term
that uses the constraints. As initial estimator we use the empirical estimator
\[
\hH = \avi h(X_i,Y_i).
\]
For each $j=0,1,\dots$ and each $k\times p$-matrix $M_j$ we obtain another
estimator as
\[
\hH_{M_j} = \avi\big(h(X_i,Y_i) 
- M_j^\top\phi_j(\hG(X_i))a_{\hvt}(X_i,Y_i)\big).
\]
Using the first $m+1$ elements of the ONB, we get the estimator
\[
\hH_m = \avi\Big(h(X_i,Y_i) 
- \sum_{j=0}^m M_j^\top\phi_j(\hG(X_i))a_{\hvt}(X_i,Y_i)\Big).
\]
It has influence function [\dots] and asymptotic variance [\dots].
The asymptotic variance is minimized by $M=M^*=\dots$.
This choice depends on the underlying distribution $P$
and must be estimated, for example by plugging in empirical estimators [\dots].
This does not change the asymptotic variance of the estimator for $E[h(X,Y)]$.
In Section 3 we set $m=m_n\to\infty$ and show that the resulting
estimator is asymptotically efficient. 






\end{document}


