Тензорный поток начинается быстро и замедляется во время тренировки

Question

Im ученик электротехники и им пытается смоделировать промышленную установку на основе мощности в резисторе внутри бурильщика, температуру воды в бойлере и поток воды, проходящий через boiller с использованием python 3.5 и tensorflow. Дело в том, что я новичок в python и tensorflow, и я написал этот код, который работает, но тренировка начинается быстро и быстро начинает замедляться, и к середине тренировки начинают начинаться между этапами. Мне просто нужна помощь в оптимизации, и, конечно же, любые советы приветствуются! Большое спасибо! Вот код:

import numpy as np 
import tensorflow as tf 
input_vec_size = 3 
step_size = 0.05 
batch_size = 3 
test_size = 16 
train_end = 1905 
eval_end = 290 
predict_end = 1396 
n_cores = 4 
def read_my_file_format(filename_queue): 
    line_reader = tf.TextLineReader(skip_header_lines=1) 
    _, csv_row = line_reader.read(filename_queue) 
    record_defaults = [[0.0], [0.0], [0.0], [0.0]] 
    time, power_in, temperature, flow = \ 
     tf.decode_csv(csv_row, record_defaults=record_defaults) 
    features = tf.pack([ 
     power_in, 
     temperature 
     ]) 
    return features, flow 
def input_pipeline(directory, batch_size, n_cores, buffer_size, num_epochs=None): 
    filename_queue = tf.train.string_input_producer(
     tf.train.match_filenames_once(directory), 
     shuffle=True) 
    features, flow = read_my_file_format(filename_queue) 
    x, y = tf.train.batch(
     [features, flow], batch_size=batch_size, allow_smaller_final_batch=True) 
def init_weights(shape): 
    return tf.Variable(tf.random_normal(shape, stddev=0.001)) 
def init_bias(shape): #inicializa bias 
    initial = tf.constant(0.001, shape=shape) #variancia 0.1 
    return tf.Variable(initial) 
def model(X, w_h, w_h2, w_o, B, B2, B3, p_keep_input, p_keep_hidden): 
    X = tf.nn.dropout(X, p_keep_input) 
    h = tf.nn.relu(tf.matmul(X, w_h)+B) 
    h = tf.nn.dropout(h, p_keep_hidden) 
    h2 = tf.nn.relu(tf.matmul(h, w_h2)+B2) 
    h2 = tf.nn.dropout(h2, p_keep_hidden) 
    return tf.matmul(h2, w_o)+B3 

X = tf.placeholder("float", [None, input_vec_size]) 
Y = tf.placeholder("float", [None, 1]) 
p_keep_hidden = tf.placeholder("float") 
p_keep_input = tf.placeholder("float") 
w_h = init_weights([input_vec_size, fclayer_size]) 
w_h2= init_weights([fclayer_size, fclayer_size]) 
w_o= init_weights([fclayer_size, 1]) 
B = init_bias([fclayer_size]) 
B2 = init_bias([fclayer_size]) 
B3 = init_bias([1]) 
py_x = model(X, w_h, w_h2, w_o, B, B2, B3, p_keep_input, p_keep_hidden) 
predict_op = py_x[0] 
cost = tf.reduce_mean(tf.square(predict_op - Y)) 
train_op = tf.train.MomentumOptimizer(step_size, 0.5).minimize(cost) 
saver = tf.train.Saver() 
directory = "./train/*.csv" 
x, y = input_pipeline(directory, batch_size, n_cores, buffer_size, num_epochs=None) 
directory_eval = "./eval/*.csv" 
xe, ye = input_pipeline(directory_eval, test_size, n_cores, buffer_size, num_epochs=None) 
directory_predict = "./predict/*.csv" 
xp, yp = input_pipeline(directory_predict, test_size, n_cores, buffer_size, num_epochs=None) 

with tf.Session() as sess: 
    tf.initialize_all_variables().run() 
    coord = tf.train.Coordinator() 
    threads = tf.train.start_queue_runners(coord=coord) 
    print("==================================TREINAMENTO=================================") 
    for iteraction in range(int(train_end/batch_size)): 
     trX, trY = sess.run([x,y]) 
     for i in range(0, batch_size): 
      features, features_past, features_past2 = sess.run(tf.unpack(trX[i])), sess.run(tf.unpack(trX[i-1])), sess.run(tf.unpack(trX[i-2])) 
     power_in_i = features[0] - 4 
     temperature_i = features[1] 
     temperature_i1 = features_past[1] 
     temperature_i2 = features_past2[1] 
     trX_now = tf.pack([power_in_i, (temperature_i-temperature_i1), (temperature_i-temperature_i2)]) 
     trX_now = sess.run(trX_now) 
     X_Batch, Y_Batch = trX_now.reshape([-1, input_vec_size]), trY[i].reshape([-1, 1]) 
     sess.run(train_op, feed_dict={X: X_Batch, 
             Y: Y_Batch, p_keep_input: 0.95, p_keep_hidden: 0.7}) 
     if(i%batch_size == 0): 
      predict_train = sess.run(tf.reshape(predict_op, [-1, 1]), feed_dict={X: X_Batch, p_keep_input: 1.0, p_keep_hidden: 1.0}) 
      train_cost = sess.run(cost, feed_dict={py_x: predict_train, Y: Y_Batch}) 
      print("Train Batch:", iteraction,"Sample:", batch_size*iteraction, "X:", X_Batch, "Y:", Y_Batch, "y_:", 
       predict_train, "Cost:", train_cost) 
    saver.save(sess, "./model.ckpt") 
    print('Variaveis salvas com sucesso') 
    coord.request_stop() 
    coord.join(threads) 
    sess.close() 
    print('=============================Fim do Treinamento=============================') 
with tf.Session() as sess: 
    tf.initialize_all_variables().run() 
    coord = tf.train.Coordinator() 
    threads = tf.train.start_queue_runners(coord=coord) 

    print("==============================VALIDAÇAO==================================") 
    saver.restore(sess, "./model.ckpt") 
    print("Model restored.") 
    for iteraction in range(int(eval_end/test_size)): 
     teX, teY = sess.run([xe, ye]) 
     for i in range(0, test_size): 
      features, features_past, features_past2 = sess.run(tf.unpack(teX[i])), sess.run(tf.unpack(teX[i - 1])), sess.run(tf.unpack(teX[i-2])) 
     power_in_i = features[0] - 4 
     temperature_i = features[1] 
     temperature_i1 = features_past[1] 
     teX_now = tf.pack([power_in_i, (temperature_i - temperature_i1), (temperature_i-temperature_i2)]) 
     teX_now = sess.run(teX_now) 
     X_Batch, Y_Batch = teX_now.reshape([-1, input_vec_size]), teY[i].reshape([-1, 1]) 
     predict_eval = sess.run(tf.reshape(predict_op, [-1, 1]), feed_dict={X: X_Batch, p_keep_input: 1.0, p_keep_hidden: 1.0}) 
     eval_cost = sess.run(cost, feed_dict={py_x: predict_eval, Y: Y_Batch}) 
     print("Eval Batch:", iteraction,"Sample:", batch_size*iteraction, "X:", X.eval(feed_dict={X: X_Batch}), "Y:", Y_Batch, "y_:", 
      predict_eval, "Cost:", eval_cost) 
    coord.request_stop() 
    coord.join(threads) 
    sess.close() 
print('=============================FIM DA VALIDAÇAO=============================') 

with tf.Session() as sess: 
    tf.initialize_all_variables().run() 
    coord = tf.train.Coordinator() 
    threads = tf.train.start_queue_runners(coord=coord) 

    print("==============================PREDIÇÃO==================================") 
    saver.restore(sess, "./model.ckpt") 
    print("Model restored.") 
    predict_batch_mean = 0 
    predict_mean = 0 
    for iteraction in range(int(predict_end/test_size)): 
     tpX, tpY = sess.run([xp, yp]) 
     for i in range(0, test_size): 
      features, features_past, features_past2 = sess.run(tf.unpack(tpX[i])), sess.run(tf.unpack(tpX[i - 1])), sess.run(tf.unpack(tpX[i-2])) 
      power_in_i = features[0]- 4 
      temperature_i = features[1] 
      temperature_i1 = features_past[1] 
      tpX_now = tf.pack([power_in_i, (temperature_i - temperature_i1), (temperature_i-temperature_i2)]) 
      tpX_now = sess.run(tpX_now) 
      X_Batch, Y_Batch = tpX_now.reshape([-1, input_vec_size]), tpY[i].reshape([-1, 1]) 
      prediction = sess.run(tf.reshape(predict_op, [-1, 1]), feed_dict={X: X_Batch, p_keep_input: 1.0, p_keep_hidden: 1.0}) 
      print("Predict Batch:", iteraction,"Sample:", batch_size*iteraction, "X:", X.eval(feed_dict={X: X_Batch}), "y_:", 
       prediction) 
      predict_batch_mean = (predict_batch_mean + prediction)/i 
     predict_mean = (predict_mean + predict_batch_mean)/iteraction 
     print("Predicted Flow:", predict_mean) 
    coord.request_stop() 
    coord.join(threads) 
    sess.close() 

Answer 1

Мои быстро догадаться, что вы создаете много новых узлы в каждой итерации через тренировку: то tf.packs и tf.reshapes просто сделать ваш график больше и больше.

Постройте график один раз за пределами тренировочного цикла, и я готов поспорить, все становится счастливым.

Answer 2

Панда спас меня на этот раз, и я уже влюбился в него! После некоторого обучения есть рабочий код. Теперь его быстро, в стороне от «не того хорошего» предсказания точности пока. Heres код:

import numpy as np 
import pandas as pd 
import tensorflow as tf 

#VARIAVEIS 
input_vec_size = 6 
layer1_size = 512 
fclayer_size = 1024 
step_size = 0.02 
test_size = 16 
train_end = 1905 
eval_end = 290 
predict_end = 1396 

#READS TRAIN FILE 
def read_data(directory): 
    data=pd.read_csv(directory, sep=',',header=None) 
    return data 

#Batch Maker 
def get_batch(data, i, data_size): 
    j = i + (input_vec_size - 1 - data_size)*(i//(data_size -  input_vec_size + 1)) + input_vec_size - 1 
# print(j, i//(data_size - 5)) 

    features = [(data[1][j] - 4)/16, 
       (data[2][j] - data[2][j - 1])*10, 
       (data[2][j] - data[2][j - 2])*10, 
       (data[2][j] - data[2][j - 3])*10, 
       (data[2][j] - data[2][j - 4])*10, 
       (data[2][j] - data[2][j - 5])*10] 
    features = np.reshape(features, [-1, input_vec_size]) 
    flow = data[3][j]/1500 
    flow = np.reshape(flow, [-1,1]) 
    return features, flow 

#Inicializaçao de variaveis 
def init_weights(shape): 
    return tf.Variable(tf.random_normal(shape, stddev=0.001)) 

def init_bias(shape): #inicializa bias 
    initial = tf.constant(0.001, shape=shape) #variancia 0.1 
    return tf.Variable(initial) 

#Definindo Modelo DNN 
def model(X, w_h, w_h2, w_o, B, B2, p_keep_input, p_keep_hidden): 
    X = tf.nn.dropout(X, p_keep_input) 
    h = tf.nn.relu(tf.matmul(X, w_h)+B) 
    h = tf.nn.dropout(h, p_keep_hidden) 
    h2 = tf.nn.relu(tf.matmul(h, w_h2)) 
    h2 = tf.nn.dropout(h2, p_keep_hidden) 
    return tf.matmul(h2, w_o) 

#PLaceholders 
X = tf.placeholder("float", [None, input_vec_size]) 
Y = tf.placeholder("float", [None, 1]) 
p_keep_hidden = tf.placeholder("float") 
p_keep_input = tf.placeholder("float") 

#Estados iniciais das variaveis 
w_h = init_weights([input_vec_size, layer1_size]) 
w_h2= init_weights([layer1_size, fclayer_size]) 
w_o= init_weights([fclayer_size, 1]) 
B = init_bias([layer1_size]) 
B2 = init_bias([fclayer_size]) 
#Modelo 
py_x = model(X, w_h, w_h2, w_o, B, B2, p_keep_input, p_keep_hidden) 
#Operaçao de previsão 
predict_op = tf.reshape(py_x[0], [-1,1]) 
#Funçao custo 
cost = tf.reduce_mean(tf.square(predict_op - Y)) 
#Operação de treinamento 
train_op = tf.train.AdadeltaOptimizer(step_size).minimize(cost) 
#Utilizado para salvar as variaveis apos o treinamento 
saver = tf.train.Saver() 


with tf.Session() as sess: 
    tf.initialize_all_variables().run() 
    directory = '~/PycharmProjects/modelagemELT430/train/G2.csv' 
    data = read_data(directory) 
    for i in range(0, 10*(train_end - input_vec_size + 1)): 
     features, flow = get_batch(data, i, train_end) 
#  features = sess.run(features) 
     sess.run(train_op, feed_dict={X: features, 
          Y: flow, p_keep_input: 0.9, p_keep_hidden: 0.6}) 
     predict_train = sess.run(predict_op, 
          feed_dict={X: features, p_keep_input: 1.0, p_keep_hidden: 1.0}) 
     train_cost = sess.run(cost, feed_dict={py_x: predict_train, Y: flow}) 
     print("Train Sample:", i, "X:", features, "Y:", flow*1500, "y_:", 
      predict_train*1500, "Cost:", train_cost) 
    saver.save(sess, "./model.ckpt") 
    print('Variaveis salvas com sucesso') 
    sess.close() 
print('=============================Fim do Treinamento=============================') 

with tf.Session() as sess: 
    tf.initialize_all_variables().run() 
    directory = '~/PycharmProjects/modelagemELT430/eval/G2E.csv' 
    data = read_data(directory) 
     print("==============================VALIDAÇAO==================================") 
    saver.restore(sess, "./model.ckpt") 
    print("Model restored.") 
    for i in range(0, eval_end - input_vec_size + 1): 
     features, flow = get_batch(data, i, eval_end) 
     predict_eval = sess.run(predict_op, 
          feed_dict={X: features, p_keep_input: 1.0, p_keep_hidden: 1.0}) 
     eval_cost = sess.run(cost, feed_dict={py_x: predict_eval, Y: flow}) 
     print("Eval Sample:", i, "X:", features, "Y:",flow*1500, "y_:",predict_eval*1500, "Cost:", eval_cost) 
    sess.close() 
print('============================Fim da Validação=================================') 

with tf.Session() as sess: 
    tf.initialize_all_variables().run() 
    directory = '~/PycharmProjects/modelagemELT430/predict/G2P.csv' 
    data = read_data(directory) 
    print("==============================Predição==================================") 
    saver.restore(sess, "./model.ckpt") 
    print("Model restored.") 
    for i in range(0, predict_end - input_vec_size + 1): 
     features, flow = get_batch(data, i, predict_end) 
     predict = sess.run(predict_op, 
          feed_dict={X: features, p_keep_input: 1.0, p_keep_hidden: 1.0}) 
     eval_cost = sess.run(cost, feed_dict={py_x: predict, Y: flow}) 
     print("Predict Sample:", i, "X:", features, "y_:",predict*1500) 
    sess.close() 
print('============================Fim da Prediçao=================================')