\documentclass[11pt]{article}
\usepackage{amsfonts}
\usepackage{amsthm}
\usepackage{amsmath}
\usepackage{latexsym}
\usepackage{verbatim}
\usepackage[pdftex]{graphicx}
\usepackage{url}
\usepackage{cancel}
\usepackage[table]{xcolor}
\usepackage[all,arc]{xy}
\usepackage{color, colortbl}
\usepackage{mathrsfs}
\usepackage{hyperref}
\hypersetup{colorlinks=true}
\setlength{\oddsidemargin}{.25in}
\setlength{\evensidemargin}{.25in}
\setlength{\textwidth}{6in}
\setlength{\topmargin}{-0.4in}
\setlength{\textheight}{8.5in}

\input{preamble.tex}

\begin{document}

\lecture{3: Orthonormal Bases}{Week 3}

In our last class, we introduced the concept of ``changing bases,'' and talked about writing vectors and linear transformations in other bases.  In the homework and in class, we saw that in several situations this idea of ``changing basis'' could make a linear transformation much easier to work with; in several cases, we saw that linear transformations under a certain basis would become diagonal, which made tasks like raising them to large powers far easier than these problems would be in the standard basis.

But how do we find these ``nice'' bases?  What does it mean for a basis to be``nice?'' In this set of lectures, we will study one potential answer to this question: the concept of an \textbf{orthonormal basis}.

\section{Orthogonality}

To start, we should define the notion of \textbf{orthogonality}.  First, recall/remember the definition of the \textbf{dot product}:

\begin{defn}
Take two vectors $(x_1, \ldots x_n), (y_1, \ldots y_n) \in \mathbb{R}^n$.  Their \textbf{dot product} is simply the sum
\begin{align*}
x_1y_1 + x_2y_2 + \ldots x_ny_n.
\end{align*}
\end{defn}
Many of you have seen an alternate, geometric definition of the dot product:
\begin{defn}
Take two vectors $(x_1, \ldots x_n), (y_1, \ldots y_n) \in \mathbb{R}^n$.  Their \textbf{dot product} is the product
\begin{align*}
||\vec{x}|| \cdot ||\vec{y}|| \cos(\theta),
\end{align*}
where $\theta$ is the angle between $\vec{x}$ and $\vec{y}$, and $||\vec{x}||$ denotes the length of the vector $\vec{x}$, i.e.\ the distance from $(x_1, \ldots x_n)$ to $(0, \ldots 0)$.
\end{defn}

These two definitions are equivalent:
\begin{thm}
Let $\vec{x} = (x_1, x_2, x_3), \vec{y} = (y_1, y_2, y_3)$ be a pair of vectors in $\mathbb{R}^3$.  Then the algebraic interpretation of $\vec{x} \cdot \vec{y}$, given by
\begin{align*}
x_1y_1 + x_2y_2 + \ldots x_ny_n,
\end{align*}
  is equal to the geometric interpretation
\begin{align*}
||\vec{x}|| \cdot ||\vec{y}|| \cos(\theta),
\end{align*}
where $\theta$ is the angle between $\vec{x}$ and $\vec{y}$.
\end{thm}
This is not a hard theorem to prove; it's essentially a consequence of some basic trigonometry, and you should do it if you're interested!  Instead, I mostly want to mention it because it lets us discuss the concept of \textbf{orthogonality}:
\begin{defn}
Two vectors $\vec{v}, \vec{w}$ are called \textbf{orthogonal} if their dot product is 0.
\end{defn}
Geometrically, we can interpret this as saying that two vectors are orthogonal whenever the angle $\theta$ between them is such that $\cos(\theta) = 0$; i.e.\ that these two vectors meet at either a $90^\circ$ or $270^\circ$-degree (i.e. $\pm \pi/2$ in radians) angle!  

\begin{center}
\includegraphics[width=2.5in]{math108b_w2014_orthvecs.pdf}\\
\small Two orthogonal vectors.  Note that their algebraic dot product $1 \cdot 3 + 3 \cdot(-1)$ is zero, and that the angle between them is $\pi/2$.
\end{center}
\normalsize


That's orthogonality!  In our introduction to these notes, we said that the idea of \textbf{orthogonality} was motivated by a desire to understand what makes a basis ``useful.''  In the next section, we look at how orthogonality and bases interact:

\section{An Orthonormal Basis: Definitions}

Consider the following definition:
\begin{defn}
Suppose we have a basis $B = \{\vec{b_1}, \ldots \vec{b_n}\}$ for some space $F^n$, where $F$ is either $\mathbb{R}$ or $\mathbb{C}$.  We will say that this basis is \textbf{orthonormal} if it satisfies the following two properties:
\begin{enumerate}
\item \textbf{Orthogonality}: if we take any $\vec{b_i}, \vec{b_j}$ with $i \neq j$, then $\vec{b_i}$ and $\vec{b_j}$ are orthogonal: i.e.\ their dot product is 0.
\item \textbf{Unit length}: the length $||\vec{b_i}||$ of any vector $\vec{b_i}$ is 1.
\end{enumerate}
\end{defn}

This is a kind of basis!  Basically, it's the best kind of basis, for the following reason: suppose that $B = \{ \vec{b_1}, \ldots \vec{b_n}\}$ is an orthonormal basis for $F^n$.  As discussed last week, we know that
\begin{align*}
A = \begin{bmatrix} \vdots & & \vdots \\ \vec{b_1} & \ldots & \vec{b_n} \\ \vdots & & \vdots \end{bmatrix}
\end{align*}
is the matrix that converts vectors written in the basis $B$ into vectors written in the standard basis, and
\begin{align*}
A^{-1} = \begin{bmatrix} \vdots & & \vdots \\ \vec{b_1} & \ldots & \vec{b_n} \\ \vdots & & \vdots \end{bmatrix}^{-1}
\end{align*}
is the matrix that converts vectors in the standard basis to vectors in the basis $B$.  Then we have the following result:
\begin{prop}
$A^{-1}$ is precisely $A^T$, the transpose\footnote{Given a $m \times n$ matrix $A$, the transpose $A^T$ of $A$ is the $n \times m$ matrix that you get by ``flipping'' $A$ over its main diagonal: i.e. by setting entry $(i,j)$ of $A^T$ to be $a_{j,i}$.  For example, the transpose of $\begin{bmatrix} 2 & 3 \\ 1 & 4 \end{bmatrix}$ is $\begin{bmatrix} 2 & 1 \\ 3 & 4 \end{bmatrix}$.} of $A$!  In other words, 
\begin{align*}
A^{-1} = \begin{bmatrix} \ldots & \vec{b_1} & \ldots \\ & \vdots & \\ \ldots & \vec{b_n} & \ldots \end{bmatrix},
\end{align*}
the matrix whose rows are given by the vectors of $B$.
\end{prop}
\begin{proof}
Just check the answer by multiplying these two matrices together!  In other words, just look at
\begin{align*}
A^T \cdot A =    \begin{bmatrix} \ldots & \vec{b_1} & \ldots \\ & \vdots & \\ \ldots & \vec{b_n} & \ldots \end{bmatrix} \cdot \begin{bmatrix} \vdots & & \vdots \\ \vec{b_1} & \ldots & \vec{b_n} \\ \vdots & & \vdots \end{bmatrix}.
\end{align*}
To get the entries of the product of these two matrices, we just take the dot product of a row of the first matrix with a column of the second matrix.  For the two matrices at hand, this means that the entry in $(i,j)$ of the product is just
\begin{align*}
\vec{b_i} \cdot \vec{b_j}.
\end{align*}
But we know these dot products!  In particular, we know that the basis $B$ is orthonormal: therefore, whenever $i \neq j$, this dot product is 0, because these two vectors are orthogonal.  As well, when $i=j$, our dot product is just
\begin{align*}
\vec{b_i} \cdot \vec{b_i} = b_{i,1} \cdot b_{i,1} + b_{i,2} \cdot b_{i,2} + \ldots + b_{i,n} \cdot b_{i,n} = \left( \sqrt{ b_{i,1}^2 + \ldots + b_{i,n}^2} \right)^2 = 1^2 = 1,
\end{align*}
because the length $\sqrt{ b_{i,1}^2 + \ldots + b_{i,n}^2}$ of each vector $\vec{b_i}$ is precisely 1.

In other words, we've just shown that the product of $A^T$ and $A$ is the matrix where $(i,j) = 0$ whenever $i \neq j$ and $(i,j) = 1$ if $i = j$; i.e.\ it's the identity matrix!  In other words, by definition we have that $A^T$ is the inverse of $A$, as multiplying $A$ by $A^T$ yields the identity matrix.
\end{proof}

Why do we care about this result?  Well: it lets us effectively calculate the matrix that transforms standard-basis vectors into basis-$B$ vectors for free!  Normally, we have to explicitly find the inverse of this matrix, which is computationally expensive and difficult.  For this nice basis, however, you just have to find the transpose of $\begin{bmatrix} \vdots & & \vdots \\ \vec{b_1} & \ldots & \vec{b_n} \\ \vdots & & \vdots \end{bmatrix}$, which is really easy!



\section{An Orthonormal Basis: Examples}

Before we do more theory, we first give a quick example of two orthonormal bases, along with their change-of-basis matrices.  

\begin{example}
One trivial example of an orthonormal basis is the \textbf{standard basis}!  In other words, look at the basis $E = \{\vec{e_1} = (1,0,0, \ldots 0), \ldots \vec{e_n} = (0, \ldots 0 ,1)\}$.  Everything in this basis trivially has length 1 and is orthogonal.
\end{example}

\begin{example}
Here are four vectors that form an orthonormal basis for $\mathbb{R}^4$: 
\begin{align*}
H = \left\{ \left( \frac12, \frac12, \frac12, \frac12\right), \left( \frac12, \frac12, -\frac{1}2, -\frac{1}2\right), \left( \frac12, -\frac12, -\frac12, \frac12 \right), \left( \frac12, -\frac12, \frac12, -\frac12 \right), \right\}.
\end{align*}
To see that they are all mutually orthogonal, it helps to factor out a $\frac12$ from these four vectors:
\begin{align*}
H = \left\{ \frac12(1,1,1,1), \frac12(1,-1,1,-1), \frac12(1,-1,-1,1),  \frac12(1,1,-1,-1)\right\}.
\end{align*}
You are invited to check that these four vectors are all orthogonal and have unit length, and thus form an orthonormal basis!  The vector that converts vectors from the basis $H$ to the standard basis is just the matrix with these vectors as its columns:
\begin{align*}
\begin{bmatrix} \frac12 & \frac12 & \frac12 & \frac12\\ \frac12 & -\frac12 & -\frac12 & \frac12 \\ \frac12 & \frac12 & -\frac12 & -\frac12 \\ \frac12 & -\frac12 & \frac12 & -\frac12 \end{bmatrix}_.
\end{align*}
By using our proposition, we can write the matrix that converts vectors from the standard basis into the basis $H$ as just the transpose of the above matrix:
\begin{align*}
\begin{bmatrix} \frac12 & \frac12 & \frac12 & \frac12\\ \frac12 & -\frac12 & \frac12 & -\frac12 \\ \frac12 & -\frac12 & -\frac12 & \frac12 \\ \frac12 & \frac12 & -\frac12 & -\frac12 \end{bmatrix}_.
\end{align*}

(This orthonormal basis has some nice properties, that students from last quarter will recognize: it's a collection of vectors such that (1) every entry in every vector has the same absolute value, and (2) they're all orthogonal!  In general, finding such sets of vectors is an open problem in mathematics, with a number of applications.  It is conjectured that for any $n$, we can find a basis for $\mathbb{R}^{4n}$ consisting of such vectors; thus far, we only know this up to $n=167$!  So, um, fun extra-extra credit problem: find one of these bases?)
\end{example}

Excellent!  We have now carefully defined the concept of an orthonormal basis, looked at some examples of orthonormal bases, and proven a theorem that explains part of why they are useful to us!  

This leaves one immediately obvious question to study: how can we make more?  
\section{Turning Bases Into Orthonormal Bases: Projection}
Specifically, let's consider the following question:
\begin{question}
Suppose that we have some $k$-dimensional vector space $V$; for example, we could have $V = \mathbb{R}^k$, or $V = \{(x_1, \ldots x_{k+1}): x_1 + \ldots x_{k+1}\}$, which you can show is a $k$-dimensional subspace of $\mathbb{R}^{k+1}$.  $V$ can be lots of things.

Can we always find an \textrm{orthonormal} basis $O_B$ for $V$, no matter what $V$ is?
\end{question}
The answer to this is \ldots not obvious at all!  So, we do what mathematicians always do when presented with a difficult problem: look at special cases!  Specifically, let's consider the simpler case where $k=2$, i.e we're dealing with a two-dimensional space $V$:

\begin{question}
Suppose that we have vector space $V$ that's two-dimensional: i.e. it has a basis $B = \{ \vec{b_1}, \vec{b_2}\}$ for a space $V$.  Can we find a new, \textbf{orthonormal} basis $O_B$ for $V$?
\end{question}
\begin{answer}
First,  make the following definition:
\begin{defn} Let $\vec{v}, \vec{w}$ be a pair of vectors in $\mathbb{R}^n$.  The \textbf{projection} of $\vec{v}$ onto $\vec{w}$, denoted $\textrm{proj}(\vec{v} \textrm{ onto } \vec{w})$, is the following vector:
\begin{itemize}
\item Take the vector $\vec{w}$.
\item Draw a line perpendicular to the vector $\vec{w}$, that goes through the point $\vec{v}$ and intersects the line spanned by the vector $\vec{w}$. 
\item $\textrm{proj}(\vec{v} \textrm{ onto } \vec{w})$ is precisely the point at which this perpendicular line intersects $\vec{w}$.
\end{itemize}
We illustrate this below:
\begin{center}
\includegraphics[width=2.5in]{mathcs103_wk8_hw_proj.pdf}
\end{center}
In particular, it bears noting that this vector is a multiple of $\vec{w}$.
\end{defn}
A formula for this vector is the following:
\begin{align*}
\textrm{proj}(\vec{v} \textrm{ onto } \vec{w}) = \frac{\vec{v} \cdot \vec{w}}{||\vec{w}||^2} \cdot \vec{w}.
\end{align*}
To see why, simply note that the vector we want is, by looking at the above picture, something of length $\cos(\theta) \cdot ||\vec{v}||$, in the direction of $\vec{w}$.  In other words,
\begin{align*}
\textrm{proj}(\vec{v} \textrm{ onto } \vec{w}) = \cos(\theta) \cdot ||\vec{v}|| \cdot \frac{\vec{w}}{||\vec{w}||}.
\end{align*}
Now, use the angle form of the dot product to see that because $\vec{w} \cdot \vec{v} = ||\vec{w}|| ||\vec{v}|| \cos(\theta)$, we have
\begin{align*}
\textrm{proj}(\vec{v} \textrm{ onto } \vec{w}) = \frac{\vec{v} \cdot \vec{w}}{||\vec{v}||||\vec{w}||}   ||\vec{v}|| \frac{\vec{w}}{||\vec{w}||}. 
\end{align*}
Canceling the $||\vec{v}||$'s gives us the desired formula.

Using this, you can define the ``orthogonal part'' of $\vec{v}$ over $\vec{w}$ in a similar fashion:
\begin{defn}
 Let $\vec{v}, \vec{w}$ be a pair of vectors in $\mathbb{R}^n$.  The \textbf{orthogonal part} of $\vec{v}$ over $\vec{w}$, denoted $\textrm{orth}(\vec{v} \textrm{ onto } \vec{w})$, is the following vector:
\begin{align*}
\textrm{orth}(\vec{v} \textrm{ over } \vec{w})  = \vec{v} - \textrm{proj}(\vec{v} \textrm{ onto } \vec{w})
\end{align*}
\end{defn}

It bears noting that this vector lives up to its name, and is in fact orthogonal to $\vec{w}$.  This is not hard to see: just take the dot product of $\vec{w}$ with it!  This yields
\begin{align*}
\vec{w} \cdot (\vec{v} - \textrm{proj}(\vec{v} \textrm{ onto } \vec{w})) &= \vec{w}\cdot \vec{v} - \vec{w} \cdot \textrm{proj}(\vec{v} \textrm{ onto } \vec{w})\\
&=\vec{w}\cdot \vec{v} -  \vec{w} \cdot(\frac{\vec{v} \cdot \vec{w}}{||\vec{w}||^2} \cdot \vec{w})\\
&= \vec{w} \cdot \vec{v} - \frac{\vec{v} \cdot\vec{w}}{||\vec{w}||^2} (\vec{w} \cdot \vec{w})\\
&= \vec{w} \cdot \vec{v} - \frac{\vec{v} \cdot\vec{w}}{||\vec{w}||^2} (w_1^2 + \ldots w_n^2)\\
&= \vec{w} \cdot \vec{v} - \frac{\vec{v} \cdot\vec{w}}{||\vec{w}||^2} (||\vec{w}||^2)\\
&= 0.\\
\end{align*}

As is often the case, actually drawing the vectors orth$(\vec{v}$ over $\vec{w})$ and $\vec{w}$ makes this proof much more believable:
\begin{center}
\includegraphics[width=3in]{math108a_finalreview_betterproj.pdf}
\end{center}

We can now answer the question that we asked at the start of this section!  Specifically: suppose that  $B = \{ \vec{b_1}, \vec{b_2}\}$ is some basis for a two-dimensional space $V$.  

We can make a new basis $O_B$ for $V$ as follows:
\begin{enumerate}
\item Define the first basis vector $\vec{r_1}'$ of $O_B$ as just $\vec{b_1}$.
\item Now, set $\vec{r_2}' = $orth$(\vec{b_2}$ over $\vec{r_1})$.  By our work above, this vector is orthogonal to $\vec{r_1}'$.
\item Finally, define  $\vec{r_1} = \vec{r_1}' / ||\vec{r_1}'||,$ $\vec{r_2} = \vec{r_2}' / ||\vec{r_2}'||.$  By construction, these vectors are just the vectors from (1) and (2), except scaled to be length 1.  
\end{enumerate}

Therefore, the set $O_B$ is made of orthonormal vectors!  We just need to show that it is an orthonormal \textbf{basis for $V$}, and we're done!  This is not hard to do: we just need to show that (1) $O_B$ spans $V$, and (2) that $O_B$ is not linearly dependent.


For (2): we will prove that in general, a set of $k$ mutually orthogonal nonzero vectors $\{\vec{r_1}, \ldots \vec{r_k}\}$ is linearly independent.  To see this, consider any linear combination of the orthogonal vectors that combines to $\vec{0}$:
\begin{align*}
a_1\vec{r_1} + \ldots a_k\vec{r_k} = \vec{0}.
\end{align*}
Take any $\vec{r_i}$, and multiply the left- and right-hand-sides by $\vec{r_i}$.  Because $\vec{r_i}, \vec{r_j}$ are orthogonal whenever $i \neq j$, the left-hand-side simplifies to $a_i \vec{r_i} \cdot \vec{r_i} = a_i \cdot ||\vec{r_i}||^2$, while the right-hand-side is just $0$.  We know that because $\vec{r_i}$ is nonzero, its length is also nonzero: therefore we can divide both sides by $||\vec{r_i}||^2$, and get that $a_i = 0$.  This holds for every $i$!  

Therefore, we have proven that the only way to combine the vectors  $\{\vec{r_1}, \ldots \vec{r_k}\}$ to get $\vec{0}$ is to use the trivial linear combination where we multiply all of these vectors by $0$.  This is the definition of linear independence!  So we've proven (2).


For (1): Look at the span of $O_B$, i.e. the set of all linear combinations of elements of $O_B$.  On one hand, because both $\vec{r_1}, \vec{r_2}$ are made out of linear combinations of the vectors $\vec{b_1}, \vec{b_2}$, the span of $O_B$ must be a subspace of $V$.  On the other hand, the span of $O_B$ is two-dimensional, because $O_B$ contains two vectors!  Therefore, the span of $O_B$ is a two-dimensional thing living in a two-dimensional space $V$: in other words, it \textbf{is} $V$.

So we're done!
\end{answer}

Here's a quick example of how the above works in practice:
\begin{example}
Examine the space $V$ spanned by the two vectors $\{ (1,2,2), (0,3,6)\}$.  Find an orthonormal basis for $V$.
\end{example}
\begin{answer}
We proceed as in our example above.  First, we set 
\begin{align*}
\vec{r_1}' =(1,2,2).
\end{align*}
Then, we define $\vec{r_2}'$ by
\begin{align*}
\vec{r_2}' &= (0,3,6) - \textrm{proj}\left((0,3,6)\textrm{ onto } (1,2,2) \right)\\
&=  (0,3,6) - \frac{\left( 0,3,6 \right) \cdot (1,2,2) }{(1,2,2)  \cdot (1,2,2) } (1,2,2) \\
&= (0,3,6) - \frac{18}{9}(1,2,2)\\
&= (-2,-1,2).
\end{align*}
Finally, we scale each by their length:
\begin{align*}
\vec{r_1} = \frac{\vec{r_1}'}{||\vec{r_1}||} = \frac{1}{\sqrt{1^2 + 2^2 + 2^2}} (1,2,2) = \left( \frac13, \frac23, \frac23 \right),\\
\vec{r_2} = \frac{\vec{r_2}'}{||\vec{r_2}||} = \frac{1}{\sqrt{(-2)^2 + (-1)^2 + 2^2}} (-2,-1,2) = \left( -\frac23, -\frac13, \frac23 \right).\\
\end{align*}
An orthonormal basis for $V$!  

You should verify for yourself that these two vectors are orthogonal, and that they can be combined to create both $(1,2,2)$ and $(0,3,6)$ (and therefore span $V$.)
\end{answer}

One more example: 
\begin{example}
Examine the space $V$ spanned by the two vectors $\{ (2,3,6), (1,0,2)\}$.  Find an orthonormal basis for $V$.
\end{example}
\begin{answer}
We proceed as in our example above.  First, we set 
\begin{align*}
\vec{r_1}' =(2,3,6).
\end{align*}
Then, we define $\vec{r_2}'$ by
\begin{align*}
\vec{r_2}' &= (1,0,2) - \textrm{proj}\left((1,0,2)\textrm{ onto } (2,3,6) \right)\\
&=  (1,0,2) - \frac{\left( 1,0,2 \right) \cdot (2,3,6) }{(2,3,6)  \cdot (2,3,6) } (2,3,6) \\
&= (1,0,2) - \frac{14}{49}(2,3,6)\\
&= (\frac37, -\frac67, \frac27).
\end{align*}
Finally, we scale each by their length:
\begin{align*}
\vec{r_1} = \frac{\vec{r_1}'}{||\vec{r_1}||} = \frac{1}{\sqrt{2^2 + 3^2 + 6^2}} (2,3,6) = \left( \frac27, \frac37, \frac67 \right),\\
\vec{r_2} = \frac{\vec{r_2}'}{||\vec{r_2}||} = \frac{1}{\sqrt{(3/7)^2 + (-6/7)^2 + (2/7)^2}}  (\frac37, -\frac67, \frac27) = (\frac37, -\frac67, \frac27).
\end{align*}
An orthonormal basis for $V$!  
\end{answer}

By thinking about how we did the proof above, we can actually get a way to answer our generalized question!  In other words, we can finally answer the following:
\begin{question}
Suppose that we have some $k$-dimensional vector space $V$; for example, we could have $V = \mathbb{R}^k$, or $V = \{(x_1, \ldots x_{k+1}): x_1 + \ldots x_{k+1}\}$, which you can show is a $k$-dimensional subspace of $\mathbb{R}^{k+1}$.  $V$ can be lots of things.

Can we always find an \textrm{orthonormal} basis $O_B$ for $V$, no matter what $V$ is?
\end{question}
\begin{answer}
We just generalize the process that we used in our work above.  Suppose that $V$ is a $k$-dimensional space with a basis $B = \{\vec{b_1}, \ldots \vec{b_k}\}$.  


Consider the following process (called the Gram-Schmidt process, formally), to create a set of $k$ nonzero orthogonal vectors $\{\vec{u_1}, \ldots \vec{u_k}\}$ out of combinations of the $B$-elements:
\begin{itemize}
\item $\vec{u_1} = \vec{b_1}.$
\item $\vec{u_2} = \vec{b_2} - \textrm{proj}(\vec{b_2} \textrm{ onto } \vec{u_1})$.  
\item $\vec{u_3} = \vec{b_3} - \textrm{proj}(\vec{b_3} \textrm{ onto } \vec{u_1}) - \textrm{proj}(\vec{b_3} \textrm{ onto } \vec{u_2})$. 
\item $\vec{u_4} = \vec{b_4} - \textrm{proj}(\vec{b_4} \textrm{ onto } \vec{u_1}) - \textrm{proj}(\vec{b_4} \textrm{ onto } \vec{u_2}) - \textrm{proj}(\vec{b_4} \textrm{ onto } \vec{u_3})$.  
\item[\vdots]
\item $\vec{u_k} = \vec{b_k} - \textrm{proj}(\vec{b_k} \textrm{ onto } \vec{u_1}) - \ldots - \textrm{proj}(\vec{b_k} \textrm{ onto } \vec{u_{k-1}})$.  
\end{itemize}

We claim that for any $n$, the vector $\vec{u_n}$ is orthogonal to any $\vec{u_m}$, with $n > m$, and prove this via strong induction.  For $n=1$, our claim is trivially true, and for $n=2$ our claim is just the special case we studied earlier; so those are  our base cases.

For the induction step: we assume that our claim holds for all $\vec{u_i}$, where $1 \leq i \leq n$.  We seek to prove our claim for $n+1$: i.e. we want to show that
\begin{align*}
\vec{u_{n+1}} = \vec{b_{n+1}} - \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_1}) - \ldots - \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_{n}})
\end{align*}
is orthogonal to every $\vec{u_j}$, for $j < n+1$.  

This is not hard to see --- just pick any value of $j$, and calculate the dot product of $\vec{u_{n+1}}$ and $\vec{u_j}$!  This is just

\begin{align*}
\vec{u_{n+1}} \cdot \vec{u_j} &= \left(\vec{b_{n+1}} - \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_1}) - \ldots - \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_{n}})\right) \cdot \vec{u_j}\\
&\begin{array}{l}=\left(\vec{b_{n+1}} - \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_j})\right) \cdot \vec{u_j}\\ \\ \\ \\ \\\end{array}  \left. \begin{array}{l} -\left( \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_1})\right) \cdot \vec{u_j}\\
 -  \left( \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_2})\right) \cdot \vec{u_j}\\
 \qquad\qquad \qquad \vdots\\
-  \left( \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_n})\right) \cdot \vec{u_j} \end{array} \right\}(\textrm{all }\vec{u_i} \neq \vec{u_j}'s).\\
\end{align*}
On one hand, we know that the first term is just
\begin{align*}
\left(\vec{b_{n+1}} - \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_j})\right) \cdot \vec{u_j} = \left( \textrm{orth}(\vec{b_{n+1}} \textrm{ over } \vec{u_j}\right) \cdot \vec{u_j},
\end{align*}
which is $0$ by construction (as these two vectors are orthogonal.)

On the other hand, each of the other terms is of the form
\begin{align*}
\left( \textrm{proj}(\vec{b_{n+1}} \textrm{ onto } \vec{u_i})\right) \cdot \vec{u_j} = (\textrm{some constant}) \vec{u_i} \cdot \vec{u_j},
\end{align*}
for $i \neq j$ and $i, j \leq n$.  We know that because $i \neq j$, we either have $i > j$ or $j > i$, and both are bounded above by $n$.  Therefore, in any case, our inductive assumption tells us that these two vectors $\vec{u_i}, \vec{u_j}$ are orthogonal!  Therefore, their dot product is also 0.

Thus, the entire dot product $\vec{u_{n+1}} \cdot \vec{u_j}$ is zero, and we have proven our inductive claim.  Consequently, we know that our claim holds for everything: i.e.\ the set $\{\vec{u_1}, \ldots \vec{u_k}\}$ is a collection of nonzero orthogonal vectors, made only out of the elements of $B$!  If we divide each of these elements by their length, we get a collection of $k$ unit-length mutually orthogonal vectors made out of the elements of $B$. 

 As proven earlier, this set is linearly independent and made out of elements of $B$ and thus spans a $k$-dimensional subspace of $V$.  But $V$ is itself $k$-dimensional: so the span of this set must be $V$ itself!  So we've found an orthonormal basis for $V$, as desired.

\end{answer}

We run one quick example here:

\begin{example}
Run the above process on the space spanned by the vectors $\{(1,1,0), (1,0,1), (0,1,1)\}$.
\end{example}
\begin{proof}
So: via the algorithm above, we define the vectors $\textbf{u}_1, \textbf{u}_2, \textbf{u}_3$ as follows:
\begin{align*}
\vec{u}_1 & = (1,1,0), \\
\vec{u}_2 & = (1,0,1)-\mathrm{proj}((1,0,1) \textrm{ onto } (1,1,0))\\
&= (1,0,1) - \frac{ (1,1,0)\cdot (1,0,1)}{ (1,1,0)\cdot (1,1,0)} (1,1,0)\\
&= (1,0,1) - \frac12 (1,1,0)\\
& = \left( \frac12, -\frac12, \frac12 \right)\\
\vec{u}_3 & = (0,1,1)-\mathrm{proj}((0,1,1) \textrm{ onto } (1,1,0))-\mathrm{proj}\left((0,1,1) \textrm{ onto } \left(\frac{1}{2},-\frac{1}{2}, \frac12\right)\right) \\
& = (0,1,1)-\frac{ (1,1,0)\cdot(0,1,1)}{ (1,1,0)\cdot (1,1,0)} (1,1,0) -\frac{ \left( \frac12, -\frac12, \frac12 \right) \cdot (0,1,1)}{ \left( \frac12, -\frac12, \frac12 \right)\cdot \left( \frac12, -\frac12, \frac12 \right)} \left( \frac12, -\frac12, \frac12 \right)\\
& = (0,1,1)-\frac12  (1,1,0) -0 \left( \frac12, -\frac12, \frac12 \right)\\
& = (0,1,1)- \left( \frac12, \frac12, 0 \right)\\
&= \left( -\frac12, \frac12, 1 \right).
\end{align*}

Fun fact: these are all now pairwise orthogonal!  Our theorem tells us this directly, but we can double-check it to illustrate the idea:
\begin{align*}
(1,1,0) \cdot\left( \frac12, -\frac12, \frac12 \right) &= -\frac12 +\frac12 = 0.\\
(1,1,0) \cdot \left( -\frac12, \frac12, 1 \right) &= -\frac12 + \frac12 = 0.\\
\left( \frac12, -\frac12, \frac12 \right) \cdot \left( -\frac12, \frac12, 1 \right) &= -\frac14 -\frac14 + \frac12= 0.\\
\end{align*}

Now, to finish up, we just scale each vector by its length:
\begin{align*}
\vec{r_1} = \frac{\vec{u_1}}{||\vec{u_1}||} = \frac{1}{\sqrt{1^2 + 1^2 + 0^2}}(1,1,0) = \left( \frac{1}{\sqrt{2}}, \frac{1}{\sqrt{2}}, 0 \right),\\
\vec{r_2} = \frac{\vec{u_2}}{||\vec{u_2}||} = \frac{1}{\sqrt{(-1/2)^2 + (1/2)^2 + 1^2}}\left( -\frac12, \frac12, 1 \right) = \left( -\frac{\sqrt{6}}{6}, \frac{\sqrt{6}}{6}, \frac{\sqrt{6}}{3} \right),\\
\vec{r_3} = \frac{\vec{u_3}}{||\vec{u_3}||} = \frac{1}{\sqrt{(1/2)^2 + (-1/2)^2 + (1/2)^2}}\left( \frac12, -\frac12, \frac12\right) = \left( \frac{\sqrt{3}}{3}, -\frac{\sqrt{3}}{3}, \frac{\sqrt{3}}{3} \right).\\
\end{align*}
Done!
\end{proof}







\end{document}