In [281]:
# generate training data
import numpy as np
N = 100
x = np.random.uniform(-1,1,N)
x = np.sort(x)
x[0]=-1
x[N-1]=1
y = (x-.95)*(x-.6)*(x-.25)*(x+.5)*(x+.9) + .03*np.random.randn(N)
# generate test data
N_ = 100
x_ = np.random.uniform(-1,1,N_)
y_ = (x_-.95)*(x_-.6)*(x_-.25)*(x_+.5)*(x_+.9) + .03*np.random.randn(N_)

t = np.linspace(-1,1,100)
y0 = (t-.95)*(t-.6)*(t-.25)*(t+.5)*(t+.9)

import matplotlib.pyplot as plt
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
plt.show()
In [282]:
# linear regression in one-shot
p=1
A = np.vstack([x, np.ones(len(x))]).T
w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
plt.plot(t, w1*t + w0, 'r')
plt.show()
yh =  w1*x + w0
MSE_train = (1/float(N))*np.sum((y - yh)**2)
y_h=  w1*x_ + w0
MSE_test = (1/float(N_))*np.sum((y_ - y_h)**2)
display(MSE_train)
display(MSE_test)
MSE = np.array([[1,MSE_train,MSE_test]])  

0.004049772488615388
0.003955706352215782
In [283]:
# linear regression in one-shot
p=2
A = np.vstack([  x**2, x, np.ones(len(x))]).T
w2, w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
plt.plot(t, w2*(t**2) + w1*t + w0, 'r')
plt.show()
yh =  w2*(x**2) + w1*x + w0
MSE_train = (1/float(N))*np.sum((y - yh)**2)
y_h=  w2*(x_**2) + w1*x_ + w0
MSE_test = (1/float(N_))*np.sum((y_ - y_h)**2)
display(MSE_train)
display(MSE_test)
MSE = np.append(MSE,[[2,MSE_train,MSE_test]],axis=0)  

0.0036258207782536816
0.003692523563652128
In [284]:
# linear regression in one-shot
p=3
A = np.vstack([ x**3, x**2, x, np.ones(len(x))]).T
w3, w2, w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
plt.plot(t, w3*(t**3) + w2*(t**2) + w1*t + w0, 'r')
plt.show()
yh =  w3*(x**3) + w2*(x**2) + w1*x + w0
MSE_train = (1/float(N))*np.sum((y - yh)**2)
y_h=  w3*(x_**3) + w2*(x_**2) + w1*x_ + w0
MSE_test = (1/float(N_))*np.sum((y_ - y_h)**2)
display(MSE_train)
display(MSE_test)
MSE = np.append(MSE,[[3,MSE_train,MSE_test]],axis=0)  

0.0036093790237525015
0.003762793591742031
In [285]:
# linear regression in one-shot
p=4
A = np.vstack([x**4, x**3, x**2, x, np.ones(len(x))]).T
w4, w3, w2, w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
t = np.linspace(-1,1,100)
plt.plot(t, w4*(t**4) + w3*(t**3) + w2*(t**2) + w1*t + w0, 'r')
plt.show()
yh = w4*(x**4) + w3*(x**3) + w2*(x**2) + w1*x + w0
MSE_train = (1/float(N))*np.sum((y - yh)**2)
y_h= w4*(x_**4) + w3*(x_**3) + w2*(x_**2) + w1*x_ + w0
MSE_test = (1/float(N_))*np.sum((y_ - y_h)**2)
display(MSE_train)
display(MSE_test)
MSE = np.append(MSE,[[p,MSE_train,MSE_test]],axis=0)  

0.0024922842606461123
0.0025716239914487667
In [286]:
# linear regression in one-shot
p=5
A = np.vstack([x**5, x**4, x**3, x**2, x, np.ones(len(x))]).T
w5, w4, w3, w2, w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
plt.plot(t, w5*(t**5) + w4*(t**4) + w3*(t**3) + w2*(t**2) + w1*t + w0, 'r')
plt.show()
yh = w5*(x**5) + w4*(x**4) + w3*(x**3) + w2*(x**2) + w1*x + w0
MSE_train = (1/float(N))*np.sum((y - yh)**2)
y_h= w5*(x_**5) + w4*(x_**4) + w3*(x_**3) + w2*(x_**2) + w1*x_ + w0
MSE_test = (1/float(N_))*np.sum((y_ - y_h)**2)
display(MSE_train)
display(MSE_test)
MSE = np.append(MSE,[[p,MSE_train,MSE_test]],axis=0)  

0.001068254273180165
0.0011100227359018184
In [287]:
# linear regression in one-shot
p=10
A = np.vstack([x**10,x**9,x**8,x**7,x**6,x**5, x**4, x**3, x**2, x, np.ones(len(x))]).T
w10,w9,w8,w7,w6,w5, w4, w3, w2, w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
plt.plot(t, w10*(t**10) + w9*(t**9) + w8*(t**8) + w7*(t**7) + w6*(t**6) + w5*(t**5) + w4*(t**4) + w3*(t**3) + w2*(t**2) + w1*t + w0, 'r')
plt.show()
yh = w10*(x**10)+w9*(x**9)+w8*(x**8)+w7*(x**7)+w6*(x**6)+w5*(x**5) + w4*(x**4) + w3*(x**3) + w2*(x**2) + w1*x + w0
MSE_train = (1/float(N))*np.sum((y - yh)**2)
y_h= w10*(x_**10)+w9*(x_**9)+w8*(x_**8)+w7*(x_**7)+w6*(x_**6)+w5*(x_**5) + w4*(x_**4) + w3*(x_**3) + w2*(x_**2) + w1*x_ + w0
MSE_test = (1/float(N_))*np.sum((y_ - y_h)**2)
display(MSE_train)
display(MSE_test)
MSE = np.append(MSE,[[p,MSE_train,MSE_test]],axis=0)  

0.0010295331769320857
0.0011164171733078461
In [288]:
# linear regression in one-shot
p=15
A = np.vstack([x**15,x**14,x**13,x**12,x**11,x**10,x**9,x**8,x**7,x**6,x**5, x**4, x**3, x**2, x, np.ones(len(x))]).T
w15,w14,w13,w12,w11,w10,w9,w8,w7,w6,w5, w4, w3, w2, w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
plt.plot(t, w15*(t**15)+w14*(t**14)+w13*(t**13)+w12*(t**12)+w11*(t**11)+w10*(t**10) + w9*(t**9) + w8*(t**8) + w7*(t**7) + w6*(t**6) + w5*(t**5) + w4*(t**4) + w3*(t**3) + w2*(t**2) + w1*t + w0, 'r')
plt.show()
yh = w15*(x**15)+w14*(x**14)+w13*(x**13)+w12*(x**12)+w11*(x**11)+w10*(x**10)+w9*(x**9)+w8*(x**8)+w7*(x**7)+w6*(x**6)+w5*(x**5) + w4*(x**4) + w3*(x**3) + w2*(x**2) + w1*x + w0
MSE_train = (1/float(N))*np.sum((y - yh)**2)
y_h= w15*(x_**15)+w14*(x_**14)+w13*(x_**13)+w12*(x_**12)+w11*(x_**11)+w10*(x_**10)+w9*(x_**9)+w8*(x_**8)+w7*(x_**7)+w6*(x_**6)+w5*(x_**5) + w4*(x_**4) + w3*(x_**3) + w2*(x_**2) + w1*x_ + w0
MSE_test = (1/float(N_))*np.sum((y_ - y_h)**2)
display(MSE_train)
display(MSE_test)
MSE = np.append(MSE,[[p,MSE_train,MSE_test]],axis=0)  

0.0009943094868201973
0.0013323875607490628
In [289]:
# linear regression in one-shot
p=20
A = np.vstack([x**20,x**19,x**18,x**17,x**16,x**15,x**14,x**13,x**12,x**11,x**10,x**9,x**8,x**7,x**6,x**5, x**4, x**3, x**2, x, np.ones(len(x))]).T
w20,w19,w18,w17,w16,w15,w14,w13,w12,w11,w10,w9,w8,w7,w6,w5, w4, w3, w2, w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
plt.plot(x,y,'o')
plt.plot(t,y0,'g-')
plt.plot(t, w20*(t**20)+w19*(t**19)+w18*(t**18)+w17*(t**17)+w16*(t**16)+w15*(t**15)+w14*(t**14)+w13*(t**13)+w12*(t**12)+w11*(t**11)+w10*(t**10) + w9*(t**9) + w8*(t**8) + w7*(t**7) + w6*(t**6) + w5*(t**5) + w4*(t**4) + w3*(t**3) + w2*(t**2) + w1*t + w0, 'r')
plt.show()
yh = w20*(x**20)+w19*(x**19)+w18*(x**18)+w17*(x**17)+w16*(x**16)+w15*(x**15)+w14*(x**14)+w13*(x**13)+w12*(x**12)+w11*(x**11)+w10*(x**10)+w9*(x**9)+w8*(x**8)+w7*(x**7)+w6*(x**6)+w5*(x**5) + w4*(x**4) + w3*(x**3) + w2*(x**2) + w1*x + w0
MSE_train = (1/float(N))*np.sum((y - yh)**2)
y_h= w20*(x_**20)+w19*(x_**19)+w18*(x_**18)+w17*(x_**17)+w16*(x_**16)+w15*(x_**15)+w14*(x_**14)+w13*(x_**13)+w12*(x_**12)+w11*(x_**11)+w10*(x_**10)+w9*(x_**9)+w8*(x_**8)+w7*(x_**7)+w6*(x_**6)+w5*(x_**5) + w4*(x_**4) + w3*(x_**3) + w2*(x_**2) + w1*x_ + w0
MSE_test = (1/float(N_))*np.sum((y_ - y_h)**2)
display(MSE_train)
display(MSE_test)
MSE = np.append(MSE,[[p,MSE_train,MSE_test]],axis=0)  

0.000884989589432562
0.0017212921468241582
In [290]:
B = np.array([[1, 0.0040792220273578, 0.003988953600984564],[2, 0.004067785006619796, 0.003944828066893552],[3, 0.004011490884146126, 0.003831010290504173],[4, 0.0019177229761741974, 0.002200492720447942],[5, 0.0007426853089970962, 0.0010239915404334238],[10, 0.0007143171682178626, 0.0010772177369285714],[15, 0.0007048906332176064, 0.0011680754928051409],[20, 0.0006350545819906561, 0.0013118370515986446]])
import matplotlib.pyplot as plt
plt.plot(MSE.T[0],MSE.T[1],'o')
plt.plot(MSE.T[0],MSE.T[2],'o')
plt.show()

why do the MSEs drop at degree 5?

what if we average over many BAD models?

In [310]:
import numpy as np
N = 80
t = np.linspace(-1,1,100)
y0 = (t-.95)*(t-.6)*(t-.25)*(t+.5)*(t+.9)
p=20
Y = np.zeros(t.shape)
Ns = 5000
for i in range(Ns):
    x = np.random.uniform(-1,1,N)
    x = np.sort(x)
    x[0]=-1
    x[N-1]=1
    y = (x-.95)*(x-.6)*(x-.25)*(x+.5)*(x+.9) + .03*np.random.randn(N)

    A = np.vstack([x**20,x**19,x**18,x**17,x**16,x**15,x**14,x**13,x**12,x**11,x**10,x**9,x**8,x**7,x**6,x**5, x**4, x**3, x**2, x, np.ones(len(x))]).T
    w20,w19,w18,w17,w16,w15,w14,w13,w12,w11,w10,w9,w8,w7,w6,w5, w4, w3, w2, w1, w0 = np.linalg.lstsq(A, y, rcond=None)[0]
    Y = Y + w20*(t**20)+w19*(t**19)+w18*(t**18)+w17*(t**17)+w16*(t**16)+w15*(t**15)+w14*(t**14)+w13*(t**13)+w12*(t**12)+w11*(t**11)+w10*(t**10) + w9*(t**9) + w8*(t**8) + w7*(t**7) + w6*(t**6) + w5*(t**5) + w4*(t**4) + w3*(t**3) + w2*(t**2) + w1*t + w0;
    if i<20:
        plt.plot(t, w20*(t**20)+w19*(t**19)+w18*(t**18)+w17*(t**17)+w16*(t**16)+w15*(t**15)+w14*(t**14)+w13*(t**13)+w12*(t**12)+w11*(t**11)+w10*(t**10) + w9*(t**9) + w8*(t**8) + w7*(t**7) + w6*(t**6) + w5*(t**5) + w4*(t**4) + w3*(t**3) + w2*(t**2) + w1*t + w0, 'r')
plt.ylim(-0.4,0.2)
plt.plot(t, (1/float(Ns))*Y, 'b')
plt.plot(t,y0,'g-')
plt.show()
In [ ]: