Как реализовать стохастический градиентный спуск на основе импульса (SGD)

Question

Я использую код python network3.py (http://neuralnetworksanddeeplearning.com/chap6.html) для разработки сверточных нейронных сетей. Теперь я хочу, чтобы изменить код немного, добавив импульс правило обучения следующим образом:

velocity = momentum_constant * velocity - learning_rate * gradient 
params = params + velocity 

Есть ли кто-нибудь знать, как сделать это? В частности, как настроить или инициализировать скорость? Я выкладываю коды для СГДА ниже:

def __init__(self, layers, mini_batch_size): 
    """Takes a list of `layers`, describing the network architecture, and 
    a value for the `mini_batch_size` to be used during training 
    by stochastic gradient descent. 

    """ 
    self.layers = layers 
    self.mini_batch_size = mini_batch_size 
    self.params = [param for layer in self.layers for param in layer.params] 
    self.x = T.matrix("x") 
    self.y = T.ivector("y") 
    init_layer = self.layers[0] 
    init_layer.set_inpt(self.x, self.x, self.mini_batch_size) 
    for j in xrange(1, len(self.layers)): 
     prev_layer, layer = self.layers[j-1], self.layers[j] 
     layer.set_inpt(
      prev_layer.output, prev_layer.output_dropout, self.mini_batch_size) 
    self.output = self.layers[-1].output 
    self.output_dropout = self.layers[-1].output_dropout 


def SGD(self, training_data, epochs, mini_batch_size, eta, 
     validation_data, test_data, lmbda=0.0): 
    """Train the network using mini-batch stochastic gradient descent.""" 
    training_x, training_y = training_data 
    validation_x, validation_y = validation_data 
    test_x, test_y = test_data 

    # compute number of minibatches for training, validation and testing 
    num_training_batches = size(training_data)/mini_batch_size 
    num_validation_batches = size(validation_data)/mini_batch_size 
    num_test_batches = size(test_data)/mini_batch_size 

    # define the (regularized) cost function, symbolic gradients, and updates 
    l2_norm_squared = sum([(layer.w**2).sum() for layer in self.layers]) 
    cost = self.layers[-1].cost(self)+\ 
      0.5*lmbda*l2_norm_squared/num_training_batches 
    grads = T.grad(cost, self.params) 
    updates = [(param, param-eta*grad) 
       for param, grad in zip(self.params, grads)] 

    # define functions to train a mini-batch, and to compute the 
    # accuracy in validation and test mini-batches. 
    i = T.lscalar() # mini-batch index 
    train_mb = theano.function(
     [i], cost, updates=updates, 
     givens={ 
      self.x: 
      training_x[i*self.mini_batch_size: (i+1)*self.mini_batch_size], 
      self.y: 
      training_y[i*self.mini_batch_size: (i+1)*self.mini_batch_size] 
     }) 
    validate_mb_accuracy = theano.function(
     [i], self.layers[-1].accuracy(self.y), 
     givens={ 
      self.x: 
      validation_x[i*self.mini_batch_size: (i+1)*self.mini_batch_size], 
      self.y: 
      validation_y[i*self.mini_batch_size: (i+1)*self.mini_batch_size] 
     }) 
    test_mb_accuracy = theano.function(
     [i], self.layers[-1].accuracy(self.y), 
     givens={ 
      self.x: 
      test_x[i*self.mini_batch_size: (i+1)*self.mini_batch_size], 
      self.y: 
      test_y[i*self.mini_batch_size: (i+1)*self.mini_batch_size] 
     }) 
    self.test_mb_predictions = theano.function(
     [i], self.layers[-1].y_out, 
     givens={ 
      self.x: 
      test_x[i*self.mini_batch_size: (i+1)*self.mini_batch_size] 
     }) 
    # Do the actual training 
    best_validation_accuracy = 0.0 
    for epoch in xrange(epochs): 
     for minibatch_index in xrange(num_training_batches): 
      iteration = num_training_batches*epoch+minibatch_index 
      if iteration % 1000 == 0: 
       print("Training mini-batch number {0}".format(iteration)) 
      cost_ij = train_mb(minibatch_index) 
      if (iteration+1) % num_training_batches == 0: 
       validation_accuracy = np.mean(
        [validate_mb_accuracy(j) for j in xrange(num_validation_batches)]) 
       print("Epoch {0}: validation accuracy {1:.2%}".format(
        epoch, validation_accuracy)) 
       if validation_accuracy >= best_validation_accuracy: 
        print("This is the best validation accuracy to date.") 
        best_validation_accuracy = validation_accuracy 
        best_iteration = iteration 
        if test_data: 
         test_accuracy = np.mean(
          [test_mb_accuracy(j) for j in xrange(num_test_batches)]) 
         print('The corresponding test accuracy is {0:.2%}'.format(
          test_accuracy)) 

Answer 1

я только когда-либо закодирован до SDG с нуля (не используя Theano), но, судя из вашего кода вы должны

1) инициируют velocities с кучей нулей (по одному на градиент),

2) включают скорость в ваших обновлениях; что-то вроде

updates = [(param, param-eta*grad +momentum_constant*vel) 
      for param, grad, vel in zip(self.params, grads, velocities)] 

3) изменить вашу функцию обучения для возврата градиентов на каждой итерации, так что вы можете обновить velocities.