神经网络的结构汇总——tflearn
一些先进的网络结构:
# https://github.com/tflearn/tflearn/blob/master/examples/images/highway_dnn.py # -*- coding: utf-8 -*- """ Deep Neural Network for MNIST dataset classification task using a highway network References: Links: [MNIST Dataset] http://yann.lecun.com/exdb/mnist/ [https://arxiv.org/abs/1505.00387](https://arxiv.org/abs/1505.00387) """ from __future__ import division, print_function, absolute_import import tflearn # Data loading and preprocessing import tflearn.datasets.mnist as mnist X, Y, testX, testY = mnist.load_data(one_hot=True) # Building deep neural network input_layer = tflearn.input_data(shape=[None, 784]) dense1 = tflearn.fully_connected(input_layer, 64, activation='elu', regularizer='L2', weight_decay=0.001) #install a deep network of highway layers highway = dense1 for i in range(10): highway = tflearn.highway(highway, 64, activation='elu', regularizer='L2', weight_decay=0.001, transform_dropout=0.8) softmax = tflearn.fully_connected(highway, 10, activation='softmax') # Regression using SGD with learning rate decay and Top-3 accuracy sgd = tflearn.SGD(learning_rate=0.1, lr_decay=0.96, decay_step=1000) top_k = tflearn.metrics.Top_k(3) net = tflearn.regression(softmax, optimizer=sgd, metric=top_k, loss='categorical_crossentropy') # Training model = tflearn.DNN(net, tensorboard_verbose=0) model.fit(X, Y, n_epoch=20, validation_set=(testX, testY), show_metric=True, run_id="highway_dense_model") # https://github.com/tflearn/tflearn/blob/master/examples/images/convnet_highway_mnist.py from __future__ import division, print_function, absolute_import import tflearn from tflearn.layers.core import input_data, dropout, fully_connected from tflearn.layers.conv import highway_conv_2d, max_pool_2d from tflearn.layers.normalization import local_response_normalization, batch_normalization from tflearn.layers.estimator import regression # Data loading and preprocessing import tflearn.datasets.mnist as mnist X, Y, testX, testY = mnist.load_data(one_hot=True) X = X.reshape([-1, 28, 28, 1]) testX = testX.reshape([-1, 28, 28, 1]) # Building convolutional network network = input_data(shape=[None, 28, 28, 1], name='input') #highway convolutions with pooling and dropout for i in range(3): for j in [3, 2, 1]: network = highway_conv_2d(network, 16, j, activation='elu') network = max_pool_2d(network, 2) network = batch_normalization(network) network = fully_connected(network, 128, activation='elu') network = fully_connected(network, 256, activation='elu') network = fully_connected(network, 10, activation='softmax') network = regression(network, optimizer='adam', learning_rate=0.01, loss='categorical_crossentropy', name='target') # Training model = tflearn.DNN(network, tensorboard_verbose=0) model.fit(X, Y, n_epoch=20, validation_set=(testX, testY), show_metric=True, run_id='convnet_highway_mnist') # https://github.com/tflearn/tflearn/blob/master/examples/images/residual_network_mnist.py from __future__ import division, print_function, absolute_import import tflearn import tflearn.data_utils as du # Data loading and preprocessing import tflearn.datasets.mnist as mnist X, Y, testX, testY = mnist.load_data(one_hot=True) X = X.reshape([-1, 28, 28, 1]) testX = testX.reshape([-1, 28, 28, 1]) X, mean = du.featurewise_zero_center(X) testX = du.featurewise_zero_center(testX, mean) # Building Residual Network net = tflearn.input_data(shape=[None, 28, 28, 1]) net = tflearn.conv_2d(net, 64, 3, activation='relu', bias=False) # Residual blocks net = tflearn.residual_bottleneck(net, 3, 16, 64) net = tflearn.residual_bottleneck(net, 1, 32, 128, downsample=True) net = tflearn.residual_bottleneck(net, 2, 32, 128) net = tflearn.residual_bottleneck(net, 1, 64, 256, downsample=True) net = tflearn.residual_bottleneck(net, 2, 64, 256) net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') net = tflearn.global_avg_pool(net) # Regression net = tflearn.fully_connected(net, 10, activation='softmax') net = tflearn.regression(net, optimizer='momentum', loss='categorical_crossentropy', learning_rate=0.1) # Training model = tflearn.DNN(net, checkpoint_path='model_resnet_mnist', max_checkpoints=10, tensorboard_verbose=0) model.fit(X, Y, n_epoch=100, validation_set=(testX, testY), show_metric=True, batch_size=256, run_id='resnet_mnist') # https://github.com/tflearn/tflearn/blob/master/examples/images/resnext_cifar10.py from __future__ import division, print_function, absolute_import import tflearn # Residual blocks # 32 layers: n=5, 56 layers: n=9, 110 layers: n=18 n = 5 # Data loading from tflearn.datasets import cifar10 (X, Y), (testX, testY) = cifar10.load_data() Y = tflearn.data_utils.to_categorical(Y) testY = tflearn.data_utils.to_categorical(testY) # Real-time data preprocessing img_prep = tflearn.ImagePreprocessing() img_prep.add_featurewise_zero_center(per_channel=True) # Real-time data augmentation img_aug = tflearn.ImageAugmentation() img_aug.add_random_flip_leftright() img_aug.add_random_crop([32, 32], padding=4) # Building Residual Network net = tflearn.input_data(shape=[None, 32, 32, 3], data_preprocessing=img_prep, data_augmentation=img_aug) net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001) net = tflearn.resnext_block(net, n, 16, 32) net = tflearn.resnext_block(net, 1, 32, 32, downsample=True) net = tflearn.resnext_block(net, n-1, 32, 32) net = tflearn.resnext_block(net, 1, 64, 32, downsample=True) net = tflearn.resnext_block(net, n-1, 64, 32) net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') net = tflearn.global_avg_pool(net) # Regression net = tflearn.fully_connected(net, 10, activation='softmax') opt = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True) net = tflearn.regression(net, optimizer=opt, loss='categorical_crossentropy') # Training model = tflearn.DNN(net, checkpoint_path='model_resnext_cifar10', max_checkpoints=10, tensorboard_verbose=0, clip_gradients=0.) model.fit(X, Y, n_epoch=200, validation_set=(testX, testY), snapshot_epoch=False, snapshot_step=500, show_metric=True, batch_size=128, shuffle=True, run_id='resnext_cifar10')
一维的,后续优化网络结构使用:
# 关于一维CNN的网络,例子较少 # https://github.com/tflearn/tflearn/blob/master/examples/nlp/cnn_sentence_classification.py # Building convolutional network network = input_data(shape=[None, 100], name='input') network = tflearn.embedding(network, input_dim=10000, output_dim=128) branch1 = conv_1d(network, 128, 3, padding='valid', activation='relu', regularizer="L2") branch2 = conv_1d(network, 128, 4, padding='valid', activation='relu', regularizer="L2") branch3 = conv_1d(network, 128, 5, padding='valid', activation='relu', regularizer="L2") network = merge([branch1, branch2, branch3], mode='concat', axis=1) network = tf.expand_dims(network, 2) network = global_max_pool(network) network = dropout(network, 0.5) network = fully_connected(network, 2, activation='softmax') network = regression(network, optimizer='adam', learning_rate=0.001, loss='categorical_crossentropy', name='target') # Training model = tflearn.DNN(network, tensorboard_verbose=0) # http://codegists.com/snippet/python/specificpy_lastzactionhero_python # Specify shape of the data, image prep network = input_data(shape=[None, 52, 64], data_preprocessing=img_prep, data_augmentation=img_aug) # conv_2d incoming, nb_filter, filter_size # incoming: Tensor. Incoming 4-D Tensor. # nb_filter: int. The number of convolutional filters. # WHAT IS THIS? # filter_size: 'intor list ofints`. Size of filters. # WHAT IS THIS? network = conv_1d(network, 512, 3, activation='relu') # (incoming, kernel_size) # incoming: Tensor. Incoming 4-D Layer. # kernel_size: 'intor list ofints`. Pooling kernel size. network = max_pool_1d(network, 2) network = conv_1d(network, 64, 3, activation='relu') network = conv_1d(network, 64, 3, activation='relu') network = max_pool_1d(network, 2) network = fully_connected(network, 512, activation='relu') network = dropout(network, 0.5) network = fully_connected(network, 4, activation='softmax') network = regression(network, optimizer='adam', loss='categorical_crossentropy', learning_rate=0.0003) model = tflearn.DNN(network, tensorboard_verbose=0) # https://github.com/gonzalolc/CharacterLevel-CNN-TFLearn/blob/master/train.py # Building convolutional network network = input_data(shape=[None, 1024], name='input') network = tflearn.embedding(network, input_dim=71, output_dim=256) network = conv_1d(network, 256, 7, padding='valid', scope='conv1', activation='relu') network = max_pool_1d(network, 3, strides=3, name='Maxpool1D_1') network = conv_1d(network, 256, 7, padding='valid', scope='conv2', activation='relu') network = max_pool_1d(network, 3, strides=3, name='Maxpool1D_2') network = conv_1d(network, 256, 3, padding='valid', scope='conv3', activation='relu') network = conv_1d(network, 256, 3, padding='valid', scope='conv4', activation='relu') network = conv_1d(network, 256, 3, padding='valid', scope='conv5', activation='relu') network = conv_1d(network, 256, 3, padding='valid', scope='conv6', activation='relu') network = max_pool_1d(network, 3, strides=3, name='Maxpool1D_Last') network = tflearn.fully_connected(network, 1024, name='Fullyconected_0') network = dropout(network, 0.5) network = tflearn.fully_connected(network, 1024, name='Fullyconected_1') network = dropout(network, 0.5) network = fully_connected(network, 14, activation='softmax', name='FullyConected_Last') network = regression(network, optimizer='adam', loss='categorical_crossentropy', name='target') # Training model = tflearn.DNN(network,tensorboard_dir='runs', checkpoint_path='checkpoints/Checkpoints', best_checkpoint_path='bestcheckpoints/BestCheckpoint', best_val_accuracy=0.94, tensorboard_verbose=2) # https://github.com/jrzaurin/Text-Classification-with-Tensorflow/blob/master/pretrained_word_embedding_TF_tflearn.py net = input_data(shape=[None,MAX_SEQUENCE_LENGTH], name='input') net = embedding(net, input_dim=MAX_NB_WORDS, output_dim=EMBEDDING_DIM, trainable=False, name="EmbeddingLayer") net = conv_1d(net, 128, 5, 1, activation='relu', padding="valid") # one could add regularization as: # net = conv_1d(net, 128, 5, 1, activation='relu', regularizer="L2", padding="valid") net = max_pool_1d(net, 5, padding="valid") net = batch_normalization(net) net = conv_1d(net, 128, 5, activation='relu', padding="valid") net = max_pool_1d(net, 5, padding="valid") net = batch_normalization(net) net = conv_1d(net, 128, 5, activation='relu', padding="valid") net = max_pool_1d(net, 35) net = batch_normalization(net) net = fully_connected(net, 128, activation='relu') net = dropout(net, 0.5) net = fully_connected(net, y_train.shape[1], activation='softmax') net = regression(net, optimizer='adam', learning_rate=0.01, loss='categorical_crossentropy', name='target') model = tflearn.DNN(net, tensorboard_verbose=0)
二维的tflearn官方的例子就非常多,到时候一维的可以借鉴他们的结构设计:
#https://github.com/tflearn/tflearn/blob/master/examples/basics/logical.py # Building a network with 2 optimizers g = tflearn.input_data(shape=[None, 2]) # Nand operator definition g_nand = tflearn.fully_connected(g, 32, activation='linear') g_nand = tflearn.fully_connected(g_nand, 32, activation='linear') g_nand = tflearn.fully_connected(g_nand, 1, activation='sigmoid') g_nand = tflearn.regression(g_nand, optimizer='sgd', learning_rate=2., loss='binary_crossentropy') # Or operator definition g_or = tflearn.fully_connected(g, 32, activation='linear') g_or = tflearn.fully_connected(g_or, 32, activation='linear') g_or = tflearn.fully_connected(g_or, 1, activation='sigmoid') g_or = tflearn.regression(g_or, optimizer='sgd', learning_rate=2., loss='binary_crossentropy') # XOR merging Nand and Or operators g_xor = tflearn.merge([g_nand, g_or], mode='elemwise_mul') # Training m = tflearn.DNN(g_xor) #https://github.com/tflearn/tflearn/blob/master/examples/images/dnn.py # Building deep neural network input_layer = tflearn.input_data(shape=[None, 784]) dense1 = tflearn.fully_connected(input_layer, 64, activation='tanh', regularizer='L2', weight_decay=0.001) dropout1 = tflearn.dropout(dense1, 0.8) dense2 = tflearn.fully_connected(dropout1, 64, activation='tanh', regularizer='L2', weight_decay=0.001) dropout2 = tflearn.dropout(dense2, 0.8) softmax = tflearn.fully_connected(dropout2, 10, activation='softmax') # Regression using SGD with learning rate decay and Top-3 accuracy sgd = tflearn.SGD(learning_rate=0.1, lr_decay=0.96, decay_step=1000) top_k = tflearn.metrics.Top_k(3) net = tflearn.regression(softmax, optimizer=sgd, metric=top_k, loss='categorical_crossentropy') # Training model = tflearn.DNN(net, tensorboard_verbose=0) # https://github.com/tflearn/tflearn/blob/master/examples/basics/finetuning.py # Redefinition of convnet_cifar10 network network = input_data(shape=[None, 32, 32, 3]) network = conv_2d(network, 32, 3, activation='relu') network = max_pool_2d(network, 2) network = dropout(network, 0.75) network = conv_2d(network, 64, 3, activation='relu') network = conv_2d(network, 64, 3, activation='relu') network = max_pool_2d(network, 2) network = dropout(network, 0.5) network = fully_connected(network, 512, activation='relu') network = dropout(network, 0.5) # Finetuning Softmax layer (Setting restore=False to not restore its weights) softmax = fully_connected(network, num_classes, activation='softmax', restore=False) regression = regression(softmax, optimizer='adam', loss='categorical_crossentropy', learning_rate=0.001) model = tflearn.DNN(regression, checkpoint_path='model_finetuning', max_checkpoints=3, tensorboard_verbose=0) # Load pre-existing model, restoring all weights, except softmax layer ones # https://github.com/tflearn/tflearn/blob/master/examples/nlp/dynamic_lstm.py # Network building net = tflearn.input_data([None, 100]) # Masking is not required for embedding, sequence length is computed prior to # the embedding op and assigned as 'seq_length' attribute to the returned Tensor. net = tflearn.embedding(net, input_dim=10000, output_dim=128) net = tflearn.lstm(net, 128, dropout=0.8, dynamic=True) net = tflearn.fully_connected(net, 2, activation='softmax') net = tflearn.regression(net, optimizer='adam', learning_rate=0.001, loss='categorical_crossentropy') # Training model = tflearn.DNN(net, tensorboard_verbose=0) # https://github.com/tflearn/tflearn/blob/master/examples/images/googlenet.py network = input_data(shape=[None, 227, 227, 3]) conv1_7_7 = conv_2d(network, 64, 7, strides=2, activation='relu', name='conv1_7_7_s2') pool1_3_3 = max_pool_2d(conv1_7_7, 3, strides=2) pool1_3_3 = local_response_normalization(pool1_3_3) conv2_3_3_reduce = conv_2d(pool1_3_3, 64, 1, activation='relu', name='conv2_3_3_reduce') conv2_3_3 = conv_2d(conv2_3_3_reduce, 192, 3, activation='relu', name='conv2_3_3') conv2_3_3 = local_response_normalization(conv2_3_3) pool2_3_3 = max_pool_2d(conv2_3_3, kernel_size=3, strides=2, name='pool2_3_3_s2') # 3a inception_3a_1_1 = conv_2d(pool2_3_3, 64, 1, activation='relu', name='inception_3a_1_1') inception_3a_3_3_reduce = conv_2d(pool2_3_3, 96, 1, activation='relu', name='inception_3a_3_3_reduce') inception_3a_3_3 = conv_2d(inception_3a_3_3_reduce, 128, filter_size=3, activation='relu', name='inception_3a_3_3') inception_3a_5_5_reduce = conv_2d(pool2_3_3, 16, filter_size=1, activation='relu', name='inception_3a_5_5_reduce') inception_3a_5_5 = conv_2d(inception_3a_5_5_reduce, 32, filter_size=5, activation='relu', name='inception_3a_5_5') inception_3a_pool = max_pool_2d(pool2_3_3, kernel_size=3, strides=1, name='inception_3a_pool') inception_3a_pool_1_1 = conv_2d(inception_3a_pool, 32, filter_size=1, activation='relu', name='inception_3a_pool_1_1') inception_3a_output = merge([inception_3a_1_1, inception_3a_3_3, inception_3a_5_5, inception_3a_pool_1_1], mode='concat', axis=3) # 3b inception_3b_1_1 = conv_2d(inception_3a_output, 128, filter_size=1, activation='relu', name='inception_3b_1_1') inception_3b_3_3_reduce = conv_2d(inception_3a_output, 128, filter_size=1, activation='relu', name='inception_3b_3_3_reduce') inception_3b_3_3 = conv_2d(inception_3b_3_3_reduce, 192, filter_size=3, activation='relu', name='inception_3b_3_3') inception_3b_5_5_reduce = conv_2d(inception_3a_output, 32, filter_size=1, activation='relu', name='inception_3b_5_5_reduce') inception_3b_5_5 = conv_2d(inception_3b_5_5_reduce, 96, filter_size=5, name='inception_3b_5_5') inception_3b_pool = max_pool_2d(inception_3a_output, kernel_size=3, strides=1, name='inception_3b_pool') inception_3b_pool_1_1 = conv_2d(inception_3b_pool, 64, filter_size=1, activation='relu', name='inception_3b_pool_1_1') inception_3b_output = merge([inception_3b_1_1, inception_3b_3_3, inception_3b_5_5, inception_3b_pool_1_1], mode='concat', axis=3, name='inception_3b_output') pool3_3_3 = max_pool_2d(inception_3b_output, kernel_size=3, strides=2, name='pool3_3_3') # 4a inception_4a_1_1 = conv_2d(pool3_3_3, 192, filter_size=1, activation='relu', name='inception_4a_1_1') inception_4a_3_3_reduce = conv_2d(pool3_3_3, 96, filter_size=1, activation='relu', name='inception_4a_3_3_reduce') inception_4a_3_3 = conv_2d(inception_4a_3_3_reduce, 208, filter_size=3, activation='relu', name='inception_4a_3_3') inception_4a_5_5_reduce = conv_2d(pool3_3_3, 16, filter_size=1, activation='relu', name='inception_4a_5_5_reduce') inception_4a_5_5 = conv_2d(inception_4a_5_5_reduce, 48, filter_size=5, activation='relu', name='inception_4a_5_5') inception_4a_pool = max_pool_2d(pool3_3_3, kernel_size=3, strides=1, name='inception_4a_pool') inception_4a_pool_1_1 = conv_2d(inception_4a_pool, 64, filter_size=1, activation='relu', name='inception_4a_pool_1_1') inception_4a_output = merge([inception_4a_1_1, inception_4a_3_3, inception_4a_5_5, inception_4a_pool_1_1], mode='concat', axis=3, name='inception_4a_output') # 4b inception_4b_1_1 = conv_2d(inception_4a_output, 160, filter_size=1, activation='relu', name='inception_4a_1_1') inception_4b_3_3_reduce = conv_2d(inception_4a_output, 112, filter_size=1, activation='relu', name='inception_4b_3_3_reduce') inception_4b_3_3 = conv_2d(inception_4b_3_3_reduce, 224, filter_size=3, activation='relu', name='inception_4b_3_3') inception_4b_5_5_reduce = conv_2d(inception_4a_output, 24, filter_size=1, activation='relu', name='inception_4b_5_5_reduce') inception_4b_5_5 = conv_2d(inception_4b_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4b_5_5') inception_4b_pool = max_pool_2d(inception_4a_output, kernel_size=3, strides=1, name='inception_4b_pool') inception_4b_pool_1_1 = conv_2d(inception_4b_pool, 64, filter_size=1, activation='relu', name='inception_4b_pool_1_1') inception_4b_output = merge([inception_4b_1_1, inception_4b_3_3, inception_4b_5_5, inception_4b_pool_1_1], mode='concat', axis=3, name='inception_4b_output') # 4c inception_4c_1_1 = conv_2d(inception_4b_output, 128, filter_size=1, activation='relu', name='inception_4c_1_1') inception_4c_3_3_reduce = conv_2d(inception_4b_output, 128, filter_size=1, activation='relu', name='inception_4c_3_3_reduce') inception_4c_3_3 = conv_2d(inception_4c_3_3_reduce, 256, filter_size=3, activation='relu', name='inception_4c_3_3') inception_4c_5_5_reduce = conv_2d(inception_4b_output, 24, filter_size=1, activation='relu', name='inception_4c_5_5_reduce') inception_4c_5_5 = conv_2d(inception_4c_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4c_5_5') inception_4c_pool = max_pool_2d(inception_4b_output, kernel_size=3, strides=1) inception_4c_pool_1_1 = conv_2d(inception_4c_pool, 64, filter_size=1, activation='relu', name='inception_4c_pool_1_1') inception_4c_output = merge([inception_4c_1_1, inception_4c_3_3, inception_4c_5_5, inception_4c_pool_1_1], mode='concat', axis=3, name='inception_4c_output') # 4d inception_4d_1_1 = conv_2d(inception_4c_output, 112, filter_size=1, activation='relu', name='inception_4d_1_1') inception_4d_3_3_reduce = conv_2d(inception_4c_output, 144, filter_size=1, activation='relu', name='inception_4d_3_3_reduce') inception_4d_3_3 = conv_2d(inception_4d_3_3_reduce, 288, filter_size=3, activation='relu', name='inception_4d_3_3') inception_4d_5_5_reduce = conv_2d(inception_4c_output, 32, filter_size=1, activation='relu', name='inception_4d_5_5_reduce') inception_4d_5_5 = conv_2d(inception_4d_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4d_5_5') inception_4d_pool = max_pool_2d(inception_4c_output, kernel_size=3, strides=1, name='inception_4d_pool') inception_4d_pool_1_1 = conv_2d(inception_4d_pool, 64, filter_size=1, activation='relu', name='inception_4d_pool_1_1') inception_4d_output = merge([inception_4d_1_1, inception_4d_3_3, inception_4d_5_5, inception_4d_pool_1_1], mode='concat', axis=3, name='inception_4d_output') # 4e inception_4e_1_1 = conv_2d(inception_4d_output, 256, filter_size=1, activation='relu', name='inception_4e_1_1') inception_4e_3_3_reduce = conv_2d(inception_4d_output, 160, filter_size=1, activation='relu', name='inception_4e_3_3_reduce') inception_4e_3_3 = conv_2d(inception_4e_3_3_reduce, 320, filter_size=3, activation='relu', name='inception_4e_3_3') inception_4e_5_5_reduce = conv_2d(inception_4d_output, 32, filter_size=1, activation='relu', name='inception_4e_5_5_reduce') inception_4e_5_5 = conv_2d(inception_4e_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_4e_5_5') inception_4e_pool = max_pool_2d(inception_4d_output, kernel_size=3, strides=1, name='inception_4e_pool') inception_4e_pool_1_1 = conv_2d(inception_4e_pool, 128, filter_size=1, activation='relu', name='inception_4e_pool_1_1') inception_4e_output = merge([inception_4e_1_1, inception_4e_3_3, inception_4e_5_5, inception_4e_pool_1_1], axis=3, mode='concat') pool4_3_3 = max_pool_2d(inception_4e_output, kernel_size=3, strides=2, name='pool_3_3') # 5a inception_5a_1_1 = conv_2d(pool4_3_3, 256, filter_size=1, activation='relu', name='inception_5a_1_1') inception_5a_3_3_reduce = conv_2d(pool4_3_3, 160, filter_size=1, activation='relu', name='inception_5a_3_3_reduce') inception_5a_3_3 = conv_2d(inception_5a_3_3_reduce, 320, filter_size=3, activation='relu', name='inception_5a_3_3') inception_5a_5_5_reduce = conv_2d(pool4_3_3, 32, filter_size=1, activation='relu', name='inception_5a_5_5_reduce') inception_5a_5_5 = conv_2d(inception_5a_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_5a_5_5') inception_5a_pool = max_pool_2d(pool4_3_3, kernel_size=3, strides=1, name='inception_5a_pool') inception_5a_pool_1_1 = conv_2d(inception_5a_pool, 128, filter_size=1, activation='relu', name='inception_5a_pool_1_1') inception_5a_output = merge([inception_5a_1_1, inception_5a_3_3, inception_5a_5_5, inception_5a_pool_1_1], axis=3, mode='concat') # 5b inception_5b_1_1 = conv_2d(inception_5a_output, 384, filter_size=1, activation='relu', name='inception_5b_1_1') inception_5b_3_3_reduce = conv_2d(inception_5a_output, 192, filter_size=1, activation='relu', name='inception_5b_3_3_reduce') inception_5b_3_3 = conv_2d(inception_5b_3_3_reduce, 384, filter_size=3, activation='relu', name='inception_5b_3_3') inception_5b_5_5_reduce = conv_2d(inception_5a_output, 48, filter_size=1, activation='relu', name='inception_5b_5_5_reduce') inception_5b_5_5 = conv_2d(inception_5b_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_5b_5_5') inception_5b_pool = max_pool_2d(inception_5a_output, kernel_size=3, strides=1, name='inception_5b_pool') inception_5b_pool_1_1 = conv_2d(inception_5b_pool, 128, filter_size=1, activation='relu', name='inception_5b_pool_1_1') inception_5b_output = merge([inception_5b_1_1, inception_5b_3_3, inception_5b_5_5, inception_5b_pool_1_1], axis=3, mode='concat') pool5_7_7 = avg_pool_2d(inception_5b_output, kernel_size=7, strides=1) pool5_7_7 = dropout(pool5_7_7, 0.4) # fc loss = fully_connected(pool5_7_7, 17, activation='softmax') network = regression(loss, optimizer='momentum', loss='categorical_crossentropy', learning_rate=0.001) # to train model = tflearn.DNN(network, checkpoint_path='model_googlenet', max_checkpoints=1, tensorboard_verbose=2) # https://github.com/tflearn/tflearn/blob/master/examples/images/densenet.py # Building Residual Network net = tflearn.input_data(shape=[None, 32, 32, 3], data_preprocessing=img_prep, data_augmentation=img_aug) net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001) net = tflearn.densenet_block(net, nb_layers, k) net = tflearn.densenet_block(net, nb_layers, k) net = tflearn.densenet_block(net, nb_layers, k) net = tflearn.global_avg_pool(net) # Regression net = tflearn.fully_connected(net, 10, activation='softmax') opt = tflearn.Nesterov(0.1, lr_decay=0.1, decay_step=32000, staircase=True) net = tflearn.regression(net, optimizer=opt, loss='categorical_crossentropy') # Training model = tflearn.DNN(net, checkpoint_path='model_densenet_cifar10', max_checkpoints=10, tensorboard_verbose=0, clip_gradients=0.) # https://github.com/AhmetHamzaEmra/tflearn/blob/master/examples/images/VGG19.py # Building 'VGG Network' input_layer = input_data(shape=[None, 224, 224, 3]) block1_conv1 = conv_2d(input_layer, 64, 3, activation='relu', name='block1_conv1') block1_conv2 = conv_2d(block1_conv1, 64, 3, activation='relu', name='block1_conv2') block1_pool = max_pool_2d(block1_conv2, 2, strides=2, name = 'block1_pool') block2_conv1 = conv_2d(block1_pool, 128, 3, activation='relu', name='block2_conv1') block2_conv2 = conv_2d(block2_conv1, 128, 3, activation='relu', name='block2_conv2') block2_pool = max_pool_2d(block2_conv2, 2, strides=2, name = 'block2_pool') block3_conv1 = conv_2d(block2_pool, 256, 3, activation='relu', name='block3_conv1') block3_conv2 = conv_2d(block3_conv1, 256, 3, activation='relu', name='block3_conv2') block3_conv3 = conv_2d(block3_conv2, 256, 3, activation='relu', name='block3_conv3') block3_conv4 = conv_2d(block3_conv3, 256, 3, activation='relu', name='block3_conv4') block3_pool = max_pool_2d(block3_conv4, 2, strides=2, name = 'block3_pool') block4_conv1 = conv_2d(block3_pool, 512, 3, activation='relu', name='block4_conv1') block4_conv2 = conv_2d(block4_conv1, 512, 3, activation='relu', name='block4_conv2') block4_conv3 = conv_2d(block4_conv2, 512, 3, activation='relu', name='block4_conv3') block4_conv4 = conv_2d(block4_conv3, 512, 3, activation='relu', name='block4_conv4') block4_pool = max_pool_2d(block4_conv4, 2, strides=2, name = 'block4_pool') block5_conv1 = conv_2d(block4_pool, 512, 3, activation='relu', name='block5_conv1') block5_conv2 = conv_2d(block5_conv1, 512, 3, activation='relu', name='block5_conv2') block5_conv3 = conv_2d(block5_conv2, 512, 3, activation='relu', name='block5_conv3') block5_conv4 = conv_2d(block5_conv3, 512, 3, activation='relu', name='block5_conv4') block4_pool = max_pool_2d(block5_conv4, 2, strides=2, name = 'block4_pool') flatten_layer = tflearn.layers.core.flatten (block4_pool, name='Flatten') fc1 = fully_connected(flatten_layer, 4096, activation='relu') dp1 = dropout(fc1, 0.5) fc2 = fully_connected(dp1, 4096, activation='relu') dp2 = dropout(fc2, 0.5) network = fully_connected(dp2, 1000, activation='rmsprop') regression = tflearn.regression(network, optimizer='adam', loss='categorical_crossentropy', learning_rate=0.001) model = tflearn.DNN(regression, checkpoint_path='vgg19', tensorboard_dir="./logs") #https://github.com/tflearn/tflearn/blob/master/examples/images/vgg_network.py # Building 'VGG Network' network = input_data(shape=[None, 224, 224, 3]) network = conv_2d(network, 64, 3, activation='relu') network = conv_2d(network, 64, 3, activation='relu') network = max_pool_2d(network, 2, strides=2) network = conv_2d(network, 128, 3, activation='relu') network = conv_2d(network, 128, 3, activation='relu') network = max_pool_2d(network, 2, strides=2) network = conv_2d(network, 256, 3, activation='relu') network = conv_2d(network, 256, 3, activation='relu') network = conv_2d(network, 256, 3, activation='relu') network = max_pool_2d(network, 2, strides=2) network = conv_2d(network, 512, 3, activation='relu') network = conv_2d(network, 512, 3, activation='relu') network = conv_2d(network, 512, 3, activation='relu') network = max_pool_2d(network, 2, strides=2) network = conv_2d(network, 512, 3, activation='relu') network = conv_2d(network, 512, 3, activation='relu') network = conv_2d(network, 512, 3, activation='relu') network = max_pool_2d(network, 2, strides=2) network = fully_connected(network, 4096, activation='relu') network = dropout(network, 0.5) network = fully_connected(network, 4096, activation='relu') network = dropout(network, 0.5) network = fully_connected(network, 17, activation='softmax') network = regression(network, optimizer='rmsprop', loss='categorical_crossentropy', learning_rate=0.0001) # Training model = tflearn.DNN(network, checkpoint_path='model_vgg', max_checkpoints=1, tensorboard_verbose=0) #https://github.com/tflearn/tflearn/blob/master/examples/images/alexnet.py # Building 'AlexNet' network = input_data(shape=[None, 227, 227, 3]) network = conv_2d(network, 96, 11, strides=4, activation='relu') network = max_pool_2d(network, 3, strides=2) network = local_response_normalization(network) network = conv_2d(network, 256, 5, activation='relu') network = max_pool_2d(network, 3, strides=2) network = local_response_normalization(network) network = conv_2d(network, 384, 3, activation='relu') network = conv_2d(network, 384, 3, activation='relu') network = conv_2d(network, 256, 3, activation='relu') network = max_pool_2d(network, 3, strides=2) network = local_response_normalization(network) network = fully_connected(network, 4096, activation='tanh') network = dropout(network, 0.5) network = fully_connected(network, 4096, activation='tanh') network = dropout(network, 0.5) network = fully_connected(network, 17, activation='softmax') network = regression(network, optimizer='momentum', loss='categorical_crossentropy', learning_rate=0.001) # Training model = tflearn.DNN(network, checkpoint_path='model_alexnet', max_checkpoints=1, tensorboard_verbose=2) #https://github.com/tflearn/tflearn/blob/master/examples/images/convnet_mnist.py # Building convolutional network network = input_data(shape=[None, 28, 28, 1], name='input') network = conv_2d(network, 32, 3, activation='relu', regularizer="L2") network = max_pool_2d(network, 2) network = local_response_normalization(network) network = conv_2d(network, 64, 3, activation='relu', regularizer="L2") network = max_pool_2d(network, 2) network = local_response_normalization(network) network = fully_connected(network, 128, activation='tanh') network = dropout(network, 0.8) network = fully_connected(network, 256, activation='tanh') network = dropout(network, 0.8) network = fully_connected(network, 10, activation='softmax') network = regression(network, optimizer='adam', learning_rate=0.01, loss='categorical_crossentropy', name='target') # Training model = tflearn.DNN(network, tensorboard_verbose=0)
在实际工作中优化过的结构(dga): ```python def get_cnn_model(max_len, volcab_size=None, is_training=True): if volcab_size is None: volcab_size = 10240000 # Building convolutional network network = tflearn.input_data(shape=[None, max_len], name='input') network = tflearn.embedding(network, input_dim=volcab_size, output_dim=32) network = conv_1d(network, 64, 3, activation='relu', regularizer="L2") network = max_pool_1d(network, 2) network = conv_1d(network, 64, 3, activation='relu', regularizer="L2") network = max_pool_1d(network, 2) #network = conv_1d(network, 64, 3, activation='relu', regularizer="L2") #network = max_pool_1d(network, 2) network = batch_normalization(network) #network = fully_connected(network, 512, activation='relu') #network = dropout(network, 0.5) network = fully_connected(network, 64, activation='relu') network = dropout(network, 0.5) network = fully_connected(network, 2, activation='softmax') #network = regression(network, optimizer='adam', learning_rate=0.01, loss='categorical_crossentropy', name='target') # Regression using SGD with learning rate decay and Top-3 accuracy sgd = tflearn.SGD(learning_rate=0.1, lr_decay=0.96, decay_step=1000) #top_k = tflearn.metrics.Top_k(3) #network = regression(network, optimizer=sgd, metric=top_k, loss='categorical_crossentropy') network = regression(network, optimizer=sgd, loss='categorical_crossentropy') model = tflearn.DNN(network, tensorboard_verbose=0) return model ``` 这是一个使用TFLearn库构建的一维卷积神经网络(1D CNN)模型。以下是这个模型的结构: 1. 输入层:接收长度为max_len的序列数据。 2. 嵌入层:将输入数据映射到一个32维的嵌入空间,词汇表大小为volcab_size。 3. 第一层卷积层:使用64个3大小的卷积核,激活函数为ReLU,正则化方法为L2。 4. 第一层最大池化层:池化窗口大小为2。 5. 第二层卷积层:使用64个3大小的卷积核,激活函数为ReLU,正则化方法为L2。 6. 第二层最大池化层:池化窗口大小为2。 7. 批量归一化层:对前一层的输出进行批量归一化。 8. 全连接层:有64个神经元,激活函数为ReLU。 9. Dropout层:Dropout比率为0.5,用于防止过拟合。 10. 输出层:有2个神经元,激活函数为softmax,用于分类。 11. 回归层:优化器为带学习率衰减的SGD,损失函数为交叉熵。 Input -> Embedding -> Conv1D -> MaxPool1D -> Conv1D -> MaxPool1D -> BatchNormalization -> FullyConnected -> Dropout -> FullyConnected -> Regression 这个模型的主要用途是文本分类或序列分类,因为它使用了嵌入层和一维卷积层。
dga应用过的!