這篇文章在前一篇文章：python構建深度神經網絡（DNN）的基礎上，添加了一下幾個內容：

1) 正則化項

2） 調出中間損失函數的輸出

3） 構建了交叉損失函數

4） 將訓練好的網絡進行保存，并調用用來測試新數據

1  數據預處理

#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2017-03-12 15:11 # @Author : CC # @File : net_load_data.py  from numpy import * import numpy as np import cPickle def load_data():  """載入解壓后的數據，并讀取"""  with open('data/mnist_pkl/mnist.pkl','rb') as f:   try:    train_data,validation_data,test_data = cPickle.load(f)    print " the file open sucessfully"    # print train_data[0].shape #(50000,784)    # print train_data[1].shape #(50000,)    return (train_data,validation_data,test_data)   except EOFError:    print 'the file open error'    return None  def data_transform():  """將數據轉化為計算格式"""  t_d,va_d,te_d = load_data()  # print t_d[0].shape # (50000,784)  # print te_d[0].shape # (10000,784)  # print va_d[0].shape # (10000,784)  # n1 = [np.reshape(x,784,1) for x in t_d[0]] # 將5萬個數據分別逐個取出化成（784,1），逐個排列  n = [np.reshape(x, (784, 1)) for x in t_d[0]] # 將5萬個數據分別逐個取出化成（784,1），逐個排列  # print 'n1',n1[0].shape  # print 'n',n[0].shape  m = [vectors(y) for y in t_d[1]] # 將5萬標簽（50000,1）化為（10,50000）  train_data = zip(n,m) # 將數據與標簽打包成元組形式  n = [np.reshape(x, (784, 1)) for x in va_d[0]] # 將5萬個數據分別逐個取出化成（784,1），排列  validation_data = zip(n,va_d[1]) # 沒有將標簽數據矢量化  n = [np.reshape(x, (784, 1)) for x in te_d[0]] # 將5萬個數據分別逐個取出化成（784,1），排列  test_data = zip(n, te_d[1]) # 沒有將標簽數據矢量化  # print train_data[0][0].shape #(784,）  # print "len(train_data[0])",len(train_data[0]) #2  # print "len(train_data[100])",len(train_data[100]) #2  # print "len(train_data[0][0])", len(train_data[0][0]) #784  # print "train_data[0][0].shape", train_data[0][0].shape #（784,1）  # print "len(train_data)", len(train_data) #50000  # print train_data[0][1].shape #(10,1)  # print test_data[0][1] # 7  return (train_data,validation_data,test_data) def vectors(y):  "賦予標簽"  label = np.zeros((10,1))  label[y] = 1.0 #浮點計算  return label 

2 網絡定義和訓練

#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2017-03-28 10:18 # @Author : CC # @File : net_network2.py  from numpy import * import numpy as np import operator import json # import sys  class QuadraticCost():  """定義二次代價函數類的方法"""  @staticmethod  def fn(a,y):   cost = 0.5*np.linalg.norm(a-y)**2   return cost  @staticmethod  def delta(z,a,y):   delta = (a-y)*sig_derivate(z)   return delta  class CrossEntroyCost():  """定義交叉熵函數類的方法"""  @staticmethod  def fn(a, y):   cost = np.sum(np.nan_to_num(-y*np.log(a)-(1-y)*np.log(1-a))) # not a number---0, inf---larger number   return cost  @staticmethod  def delta(z, a, y):   delta = (a - y)   return delta  class Network(object):  """定義網絡結構和方法"""  def __init__(self,sizes,cost):   self.num_layer = len(sizes)   self.sizes = sizes   self.cost = cost   # print "self.cost.__name__:",self.cost.__name__ # CrossEntropyCost   self.default_weight_initializer()  def default_weight_initializer(self):   """權值初始化"""   self.bias = [np.random.rand(x, 1) for x in self.sizes[1:]]   self.weight = [np.random.randn(y, x)/float(np.sqrt(x)) for (x, y) in zip(self.sizes[:-1], self.sizes[1:])]   def large_weight_initializer(self):   """權值另一種初始化"""   self.bias = [np.random.rand(x, 1) for x in self.sizes[1:]]   self.weight = [np.random.randn(y, x) for x, y in zip(self.sizes[:-1], self.sizes[1:])]  def forward(self,a):   """forward the network"""   for w,b in zip(self.weight,self.bias):    a=sigmoid(np.dot(w,a)+b)   return a   def SGD(self,train_data,min_batch_size,epochs,eta,test_data=False,    lambd = 0,    monitor_train_cost = False,    monitor_train_accuracy = False,    monitor_test_cost=False,    monitor_test_accuracy=False    ):   """1)Set the train_data,shuffle;    2) loop the epoches,    3) set the min_batches,and rule of update"""   if test_data: n_test=len(test_data)   n = len(train_data)   for i in xrange(epochs):    random.shuffle(train_data)    min_batches = [train_data[k:k+min_batch_size] for k in xrange(0,n,min_batch_size)]     for min_batch in min_batches: # 每次提取一個批次的樣本     self.update_minbatch_parameter(min_batch,eta,lambd,n)    train_cost = []    if monitor_train_cost:     cost1 = self.total_cost(train_data,lambd,cont=False)     train_cost.append(cost1)     print "epoche {0},train_cost: {1}".format(i,cost1)    if monitor_train_accuracy:     accuracy = self.accuracy(train_data,cont=True)     train_cost.append(accuracy)     print "epoche {0}/{1},train_accuracy: {2}".format(i,epochs,accuracy)    test_cost = []    if monitor_test_cost:     cost1 = self.total_cost(test_data,lambd)     test_cost.append(cost1)     print "epoche {0},test_cost: {1}".format(i,cost1)    test_accuracy = []    if monitor_test_accuracy:     accuracy = self.accuracy(test_data)     test_cost.append(accuracy)     print "epoche:{0}/{1},test_accuracy:{2}".format(i,epochs,accuracy)   self.save(filename= "net_save") #保存網絡網絡參數   def total_cost(self,train_data,lambd,cont=True):   cost1 = 0.0   for x,y in train_data:    a = self.forward(x)    if cont: y = vectors(y) #將測試樣本標簽化為矩陣    cost1 += (self.cost).fn(a,y)/len(train_data)   cost1 += lambd/len(train_data)*np.sum(np.linalg.norm(weight)**2 for weight in self.weight) #加上權值項   return cost1  def accuracy(self,train_data,cont=False):   if cont:    output1 = [(np.argmax(self.forward(x)),np.argmax(y)) for (x,y) in train_data]   else:    output1 = [(np.argmax(self.forward(x)), y) for (x, y) in train_data]   return sum(int(out1 == y) for (out1, y) in output1)  def update_minbatch_parameter(self,min_batch, eta,lambd,n):   """1) determine the weight and bias    2) calculate the the delta    3) update the data """   able_b = [np.zeros(b.shape) for b in self.bias]   able_w=[np.zeros(w.shape) for w in self.weight]   for x,y in min_batch: #每次只取一個樣本？    deltab,deltaw = self.backprop(x,y)    able_b =[a_b+dab for a_b, dab in zip(able_b,deltab)] #實際上對dw,db做批次累加，最后小批次取平均    able_w = [a_w + daw for a_w, daw in zip(able_w, deltaw)]   self.weight = [weight - eta * (dw) / len(min_batch)- eta*(lambd*weight)/n for weight, dw in zip(self.weight,able_w) ]   #增加正則化項：eta*lambda/m *weight   self.bias = [bias - eta * db / len(min_batch) for bias, db in zip(self.bias, able_b)]   def backprop(self,x,y):   """" 1) clacu the forward value    2) calcu the delta： delta =(y-f(z)); deltak = delta*w(k)*fz(k-1)'    3) clacu the delta in every layer: deltab=delta; deltaw=delta*fz(k-1)"""   deltab = [np.zeros(b.shape) for b in self.bias]   deltaw = [np.zeros(w.shape) for w in self.weight]   zs = []   activate = x   activates = [x]   for w,b in zip(self.weight,self.bias):    z =np.dot(w, activate) +b    zs.append(z)    activate = sigmoid(z)    activates.append(activate)    # backprop   delta = self.cost.delta(zs[-1],activates[-1],y) #調用不同代價函數的方法求梯度   deltab[-1] = delta   deltaw[-1] = np.dot(delta ,activates[-2].transpose())   for i in xrange(2,self.num_layer):    z = zs[-i]    delta = np.dot(self.weight[-i+1].transpose(),delta)* sig_derivate(z)    deltab[-i] = delta    deltaw[-i] = np.dot(delta,activates[-i-1].transpose())   return (deltab,deltaw)   def save(self,filename):   """將訓練好的網絡采用json(java script object notation)將對象保存成字符串保存，用于生產部署   encoder=json.dumps(data)   python 原始類型(沒有數組類型)向 json 類型的轉化對照表：    python    json    dict    object   list/tuple   arrary   int/long/float  number   .tolist() 將數組轉化為列表   >>> a = np.array([[1, 2], [3, 4]])   >>> list(a)   [array([1, 2]), array([3, 4])]   >>> a.tolist()   [[1, 2], [3, 4]]   """   data = {"sizes": self.sizes,"weight": [weight.tolist() for weight in self.weight],     "bias": ([bias.tolist() for bias in self.bias]),     "cost": str(self.cost.__name__)}   # 保存網絡訓練好的權值，偏置，交叉熵參數。   f = open(filename, "w")   json.dump(data,f)   f.close()  def load_net(filename):  """采用data=json.load(json.dumps(data))進行解碼，  decoder = json.load(encoder)  編碼后和解碼后鍵不會按照原始data的鍵順序排列，但每個鍵對應的值不會變  載入訓練好的網絡用于測試"""  f = open(filename,"r")  data = json.load(f)  f.close()  # print "data[cost]", getattr(sys.modules[__name__], data["cost"])#獲得屬性__main__.CrossEntropyCost  # print "data[cost]", data["cost"], data["sizes"]  net = Network(data["sizes"], cost=data["cost"]) #網絡初始化  net.weight = [np.array(w) for w in data["weight"]] #賦予訓練好的權值,并將list--->array  net.bias = [np.array(b) for b in data["bias"]]  return net  def sig_derivate(z):  """derivate sigmoid"""  return sigmoid(z) * (1-sigmoid(z))  def sigmoid(x):  sigm=1.0/(1.0+exp(-x))  return sigm  def vectors(y):  """賦予標簽"""  label = np.zeros((10,1))  label[y] = 1.0 #浮點計算  return label