py的源码看起来还是很愉快的。。。（虽然熟练成程度完全不如cpp。。。。
datasets里是数据集相关
deployment是部署相关
nets里给了很多网络结构
preprocessing给了几种预处理的方式
这些都和slim没有太大关系，就不多废话了。
分析的部分见代码注释…
由于刚刚入门machine learning 一周…还有很多内容还没有从理论层面接触…所以源码的理解也十分有限…希望能以后有机会补充一波
1 # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 # ==============================================================================
15 r"""Downloads and converts a particular dataset.
16
17 Usage:
18
19$ python download_and_convert_data.py \
20 --dataset_name=mnist \
21 --dataset_dir=/tmp/mnist
22
23$ python download_and_convert_data.py \
24 --dataset_name=cifar10 \
25 --dataset_dir=/tmp/cifar10
26
27$ python download_and_convert_data.py \
28 --dataset_name=flowers \
29 --dataset_dir=/tmp/flowers
30 """
31 from __future__ import absolute_import #from __future__是为了解决python版本升级导致的兼容问题，没必要纠结
32 from __future__ import division
33 from __future__ import print_function
34
35 import tensorflow as tf
36
37 from datasets import download_and_convert_cifar10
38 from datasets import download_and_convert_flowers
39 from datasets import download_and_convert_mnist
40
41 FLAGS = tf.app.flags.FLAGS #FLAGS 用来传递或者设置tensforflow的参数
42
43 tf.app.flags.DEFINE_string( #设置的格式为：('参数名称',参数值,'参数的解释'）
44 'dataset_name',
45 None,
46 'The name of the dataset to convert, one of "cifar10", "flowers", "mnist".')
47
48 tf.app.flags.DEFINE_string(
49 'dataset_dir',
50 None,
51 'The directory where the output TFRecords and temporary files are saved.')
52
53
54 def main(_):
55 if not FLAGS.dataset_name:
56 raise ValueError('You must supply the dataset name with --dataset_name')
57 if not FLAGS.dataset_dir:
58 raise ValueError('You must supply the dataset directory with --dataset_dir')
59
60 if FLAGS.dataset_name == 'cifar10': #提供的三个数据集,[cifar10],[flowers],[mnist]
61 download_and_convert_cifar10.run(FLAGS.dataset_dir)
62 elif FLAGS.dataset_name == 'flowers':
63 download_and_convert_flowers.run(FLAGS.dataset_dir)
64 elif FLAGS.dataset_name == 'mnist':
65 download_and_convert_mnist.run(FLAGS.dataset_dir)
66 else:
67 raise ValueError(
68 'dataset_name [%s] was not recognized.' % FLAGS.dataset_name) #数据经名字不属于上述三个
69
70 if __name__ == '__main__': #这种写法可以保证在该文件被import的时候不会执行main函数
71 tf.app.run()
72
73
74 # coding=utf-8
75 # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
76 #
77 # Licensed under the Apache License, Version 2.0 (the "License");
78 # you may not use this file except in compliance with the License.
79 # You may obtain a copy of the License at
80 #
81 # http://www.apache.org/licenses/LICENSE-2.0
82 #
83 # Unless required by applicable law or agreed to in writing, software
84 # distributed under the License is distributed on an "AS IS" BASIS,
85 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
86 # See the License for the specific language governing permissions and
87 # limitations under the License.
88 # ==============================================================================
89 """Generic evaluation script that evaluates a model using a given dataset."""
90
91 from __future__ import absolute_import
92 from __future__ import division
93 from __future__ import print_function
94
95 import math
96 import tensorflow as tf
97
98 from datasets import dataset_factory
99 from nets import nets_factory
100 from preprocessing import preprocessing_factory
101
102 slim = tf.contrib.slim
103
104 tf.app.flags.DEFINE_integer(
105 'batch_size', 100, 'The number of samples in each batch.')
106
107 tf.app.flags.DEFINE_integer(
108 'max_num_batches', None,
109 'Max number of batches to evaluate by default use all.')
110
111 tf.app.flags.DEFINE_string(
112 'master', '', 'The address of the TensorFlow master to use.')
113
114 tf.app.flags.DEFINE_string(
115 'checkpoint_path', '/tmp/tfmodel/',
116 'The directory where the model was written to or an absolute path to a '
117 'checkpoint file.')
118
119 tf.app.flags.DEFINE_string(
120 'eval_dir', '/tmp/tfmodel/', 'Directory where the results are saved to.')
121
122 tf.app.flags.DEFINE_integer(
123 'num_preprocessing_threads', 4,
124 'The number of threads used to create the batches.')
125
126 tf.app.flags.DEFINE_string(
127 'dataset_name', 'imagenet', 'The name of the dataset to load.')
128
129 tf.app.flags.DEFINE_string(
130 'dataset_split_name', 'test', 'The name of the train/test split.')
131
132 tf.app.flags.DEFINE_string(
133 'dataset_dir', None, 'The directory where the dataset files are stored.')
134
135 tf.app.flags.DEFINE_integer(
136 'labels_offset', 0,
137 'An offset for the labels in the dataset. This flag is primarily used to '
138 'evaluate the VGG and ResNet architectures which do not use a background '
139 'class for the ImageNet dataset.')
140
141 tf.app.flags.DEFINE_string(
142 'model_name', 'inception_v3', 'The name of the architecture to evaluate.')
143
144 tf.app.flags.DEFINE_string(
145 'preprocessing_name', None, 'The name of the preprocessing to use. If left '
146 'as `None`, then the model_name flag is used.')
147
148 tf.app.flags.DEFINE_float(
149 'moving_average_decay', None,
150 'The decay to use for the moving average.'
151 'If left as None, then moving averages are not used.')
152
153 tf.app.flags.DEFINE_integer(
154 'eval_image_size', None, 'Eval image size')
155
156 FLAGS = tf.app.flags.FLAGS
157
158
159 def main(_):
160 if not FLAGS.dataset_dir:
161 raise ValueError('You must supply the dataset directory with --dataset_dir')
162
163 tf.logging.set_verbosity(tf.logging.INFO) #设置log信息的级别，有DEBUG, INFO, WARN, ERROR, or FATAL
164 with tf.Graph().as_default(): #overrides the current default graph for the lifetime of the context
165 #注意不是线程安全的..
166 tf_global_step = slim.get_or_create_global_step()
167 #slim.get_or_create_global_step可以参考tf.train.get_or_create_global_step
168 #作用同样是得到global step tensor，参数为graph,参数为空时认为参数为default graph
169 ######################
170 # Select the dataset #
171 ######################
172 dataset = dataset_factory.get_dataset(
173 FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir)
174
175 ####################
176 # Select the model #
177 ####################
178 network_fn = nets_factory.get_network_fn(
179 FLAGS.model_name,
180 num_classes=(dataset.num_classes - FLAGS.labels_offset),
181 is_training=False)
182
183 ##############################################################
184 # Create a dataset provider that loads data from the dataset #
185 ##############################################################
186 #读取数据
187 provider = slim.dataset_data_provider.DatasetDataProvider(
188 dataset,
189 shuffle=False,
190 common_queue_capacity=2 * FLAGS.batch_size,
191 common_queue_min=FLAGS.batch_size)
192 [image, label] = provider.get(['image', 'label'])
193 label -= FLAGS.labels_offset
194
195 #####################################
196 # Select the preprocessing function #
197 #####################################
198 preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
199 image_preprocessing_fn = preprocessing_factory.get_preprocessing(
200 preprocessing_name,
201 is_training=False)
202 #python or的用法，flag1 or flag2 or... or flagn,如果最后逻辑值为真，
203 # 返回的是（葱左至右）第一个使其为真的值（而不返回布尔值），
204 #如果都为假，则返回最后一个假值
205 eval_image_size = FLAGS.eval_image_size or network_fn.default_image_size
206
207 image = image_preprocessing_fn(image, eval_image_size, eval_image_size)
208
209 images, labels = tf.train.batch(
210 [image, label],
211 batch_size=FLAGS.batch_size,
212 num_threads=FLAGS.num_preprocessing_threads,
213 capacity=5 * FLAGS.batch_size)
214
215 ####################
216 # Define the model #
217 ####################
218 logits, _ = network_fn(images) #python语法,序列解包
219
220 #移动平均，参考 https://en.wikipedia.org/wiki/Moving_average
221 if FLAGS.moving_average_decay:
222 variable_averages = tf.train.ExponentialMovingAverage(
223 FLAGS.moving_average_decay, tf_global_step)
224 variables_to_restore = variable_averages.variables_to_restore(
225 slim.get_model_variables())
226 variables_to_restore[tf_global_step.op.name] = tf_global_step
227 else:
228 variables_to_restore = slim.get_variables_to_restore()
229
230 predictions = tf.argmax(logits, 1)
231 #tf.argmax(input, axis=None, name=None, dimension=None)
232 #Returns the index with the largest value across axes of a tensor.
233 #就是返回logits的第一维（行？）最大值的位置索引
234
235
236 labels = tf.squeeze(labels) #将labels中维度是1的那一维去掉
237
238 # Define the metrics:
239 names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
240 'Accuracy': slim.metrics.streaming_accuracy(predictions, labels),
241 'Recall_5': slim.metrics.streaming_recall_at_k(
242 logits, labels, 5),
243 })
244 #metrics.aggregate_metric_map在metrics的list很长的时候的一种简便的表达方式
245 #metrics直接翻译为【度量】，不是tensorflow的概念.用来监控计算的性能指标
246
247
248 # Print the summaries to screen.
249 for name, value in names_to_values.items():
250 summary_name = 'eval/%s' % name
251 op = tf.summary.scalar(summary_name, value, collections=[])
252 #tf.summary.scalar :Outputs a Summary protocol buffer containing a single scalar value
253 op = tf.Print(op, [value], summary_name)
254 tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
255
256 # TODO(sguada) use num_epochs=1
257 if FLAGS.max_num_batches:
258 num_batches = FLAGS.max_num_batches
259 else:
260 # This ensures that we make a single pass over all of the data.
261 num_batches = math.ceil(dataset.num_samples / float(FLAGS.batch_size))
262
263 if tf.gfile.IsDirectory(FLAGS.checkpoint_path):#返回是否为一个目录
264 checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
265 else:
266 checkpoint_path = FLAGS.checkpoint_path
267
268 tf.logging.info('Evaluating %s' % checkpoint_path) #记录log信息
269
270 #Evaluates the model at the given checkpoint path.
271 #Evaluates the model at the given checkpoint path.
272 slim.evaluation.evaluate_once(
273 master=FLAGS.master,
274 checkpoint_path=checkpoint_path,
275 logdir=FLAGS.eval_dir,
276 num_evals=num_batches,
277 eval_op=list(names_to_updates.values()),
278 variables_to_restore=variables_to_restore)
279
280
281 if __name__ == '__main__':
282 tf.app.run()
283
284
285 # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
286 #
287 # Licensed under the Apache License, Version 2.0 (the "License");
288 # you may not use this file except in compliance with the License.
289 # You may obtain a copy of the License at
290 #
291 # http://www.apache.org/licenses/LICENSE-2.0
292 #
293 # Unless required by applicable law or agreed to in writing, software
294 # distributed under the License is distributed on an "AS IS" BASIS,
295 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
296 # See the License for the specific language governing permissions and
297 # limitations under the License.
298 # ==============================================================================
299 r"""Saves out a GraphDef containing the architecture of the model.
300
301 To use it, run something like this, with a model name defined by slim:
302
303 bazel build tensorflow_models/slim:export_inference_graph
304 bazel-bin/tensorflow_models/slim/export_inference_graph \
305 --model_name=inception_v3 --output_file=/tmp/inception_v3_inf_graph.pb
306
307 If you then want to use the resulting model with your own or pretrained
308 checkpoints as part of a mobile model, you can run freeze_graph to get a graph
309 def with the variables inlined as constants using:
310
311 bazel build tensorflow/python/tools:freeze_graph
312 bazel-bin/tensorflow/python/tools/freeze_graph \
313 --input_graph=/tmp/inception_v3_inf_graph.pb \
314 --input_checkpoint=/tmp/checkpoints/inception_v3.ckpt \
315 --input_binary=true --output_graph=/tmp/frozen_inception_v3.pb \
316 --output_node_names=InceptionV3/Predictions/Reshape_1
317
318 The output node names will vary depending on the model, but you can inspect and
319 estimate them using the summarize_graph tool:
320
321 bazel build tensorflow/tools/graph_transforms:summarize_graph
322 bazel-bin/tensorflow/tools/graph_transforms/summarize_graph \
323 --in_graph=/tmp/inception_v3_inf_graph.pb
324
325 To run the resulting graph in C++, you can look at the label_image sample code:
326
327 bazel build tensorflow/examples/label_image:label_image
328 bazel-bin/tensorflow/examples/label_image/label_image \
329 --image=${HOME}/Pictures/flowers.jpg \
330 --input_layer=input \
331 --output_layer=InceptionV3/Predictions/Reshape_1 \
332 --graph=/tmp/frozen_inception_v3.pb \
333 --labels=/tmp/imagenet_slim_labels.txt \
334 --input_mean=0 \
335 --input_std=255
336
337 """
338
339 from __future__ import absolute_import
340 from __future__ import division
341 from __future__ import print_function
342
343 import tensorflow as tf
344
345 from tensorflow.python.platform import gfile
346 from datasets import dataset_factory
347 from nets import nets_factory
348
349
350 slim = tf.contrib.slim
351
352 tf.app.flags.DEFINE_string(
353 'model_name', 'inception_v3', 'The name of the architecture to save.')
354
355 tf.app.flags.DEFINE_boolean(
356 'is_training', False,
357 'Whether to save out a training-focused version of the model.')
358
359 tf.app.flags.DEFINE_integer(
360 'default_image_size', 224,
361 'The image size to use if the model does not define it.')
362
363 tf.app.flags.DEFINE_string('dataset_name', 'imagenet',
364 'The name of the dataset to use with the model.')
365
366 tf.app.flags.DEFINE_integer(
367 'labels_offset', 0,
368 'An offset for the labels in the dataset. This flag is primarily used to '
369 'evaluate the VGG and ResNet architectures which do not use a background '
370 'class for the ImageNet dataset.')
371
372 tf.app.flags.DEFINE_string(
373 'output_file', '', 'Where to save the resulting file to.')
374
375 tf.app.flags.DEFINE_string(
376 'dataset_dir', '', 'Directory to save intermediate dataset files to')
377
378 FLAGS = tf.app.flags.FLAGS
379
380
381 def main(_):
382 if not FLAGS.output_file:
383 raise ValueError('You must supply the path to save to with --output_file')
384 tf.logging.set_verbosity(tf.logging.INFO)
385 with tf.Graph().as_default() as graph:
386 dataset = dataset_factory.get_dataset(FLAGS.dataset_name, 'train',
387 FLAGS.dataset_dir)
388 network_fn = nets_factory.get_network_fn(
389 FLAGS.model_name,
390 num_classes=(dataset.num_classes - FLAGS.labels_offset),
391 is_training=FLAGS.is_training)
392
393 #hasattr 是python语法， hasattr(object, name) -> bool，用来判断object中是否有name属性
394 if hasattr(network_fn, 'default_image_size'):
395 image_size = network_fn.default_image_size
396 else:
397 image_size = FLAGS.default_image_size
398 placeholder = tf.placeholder(name='input', dtype=tf.float32,
399 shape=[1, image_size, image_size, 3])
400 network_fn(placeholder)
401 graph_def = graph.as_graph_def()
402 #graph.as_graph_def():Returns a serialized（序列化） GraphDef representation of this graph.
403 #The serialized GraphDef can be imported into another Graph (using tf.import_graph_def) or used with the C++ Session API.
404 #该方法线程安全
405
406
407 # gfile。GFile 是一个无线程锁的I/O 封装
408 with gfile.GFile(FLAGS.output_file, 'wb') as f:
409 f.write(graph_def.SerializeToString())
410
411
412 if __name__ == '__main__':
413 tf.app.run()
414
415
416
417
418 # coding=utf-8
419 # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
420 #
421 # Licensed under the Apache License, Version 2.0 (the "License");
422 # you may not use this file except in compliance with the License.
423 # You may obtain a copy of the License at
424 #
425 # http://www.apache.org/licenses/LICENSE-2.0
426 #
427 # Unless required by applicable law or agreed to in writing, software
428 # distributed under the License is distributed on an "AS IS" BASIS,
429 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
430 # See the License for the specific language governing permissions and
431 # limitations under the License.
432 # ==============================================================================
433 """Generic training script that trains a model using a given dataset."""
434
435 from __future__ import absolute_import
436 from __future__ import division
437 from __future__ import print_function
438
439 import tensorflow as tf
440
441 from datasets import dataset_factory
442 from deployment import model_deploy
443 from nets import nets_factory
444 from preprocessing import preprocessing_factory
445
446 slim = tf.contrib.slim
447
448 tf.app.flags.DEFINE_string(
449 'master', '', 'The address of the TensorFlow master to use.')
450
451 tf.app.flags.DEFINE_string(
452 'train_dir', '/tmp/tfmodel/',
453 'Directory where checkpoints and event logs are written to.')
454
455 tf.app.flags.DEFINE_integer('num_clones', 1,
456 'Number of model clones to deploy.')
457
458 tf.app.flags.DEFINE_boolean('clone_on_cpu', False,
459 'Use CPUs to deploy clones.')
460
461 tf.app.flags.DEFINE_integer('worker_replicas', 1, 'Number of worker replicas.')
462
463 tf.app.flags.DEFINE_integer(
464 'num_ps_tasks', 0,
465 'The number of parameter servers. If the value is 0, then the parameters '
466 'are handled locally by the worker.')
467
468 tf.app.flags.DEFINE_integer(
469 'num_readers', 4,
470 'The number of parallel readers that read data from the dataset.')
471
472 tf.app.flags.DEFINE_integer(
473 'num_preprocessing_threads', 4,
474 'The number of threads used to create the batches.')
475
476 tf.app.flags.DEFINE_integer(
477 'log_every_n_steps', 10,
478 'The frequency with which logs are print.')
479
480 tf.app.flags.DEFINE_integer(
481 'save_summaries_secs', 600,
482 'The frequency with which summaries are saved, in seconds.')
483
484 tf.app.flags.DEFINE_integer(
485 'save_interval_secs', 600,
486 'The frequency with which the model is saved, in seconds.')
487
488 tf.app.flags.DEFINE_integer(
489 'task', 0, 'Task id of the replica running the training.')
490
491 ######################
492 # Optimization Flags #
493 ######################
494
495 tf.app.flags.DEFINE_float(
496 'weight_decay', 0.00004, 'The weight decay on the model weights.')
497
498 tf.app.flags.DEFINE_string(
499 'optimizer', 'rmsprop',
500 'The name of the optimizer, one of "adadelta", "adagrad", "adam",'
501 '"ftrl", "momentum", "sgd" or "rmsprop".')
502
503 tf.app.flags.DEFINE_float(
504 'adadelta_rho', 0.95,
505 'The decay rate for adadelta.')
506
507 tf.app.flags.DEFINE_float(
508 'adagrad_initial_accumulator_value', 0.1,
509 'Starting value for the AdaGrad accumulators.')
510
511 tf.app.flags.DEFINE_float(
512 'adam_beta1', 0.9,
513 'The exponential decay rate for the 1st moment estimates.')
514
515 tf.app.flags.DEFINE_float(
516 'adam_beta2', 0.999,
517 'The exponential decay rate for the 2nd moment estimates.')
518
519 tf.app.flags.DEFINE_float('opt_epsilon', 1.0, 'Epsilon term for the optimizer.')
520
521 tf.app.flags.DEFINE_float('ftrl_learning_rate_power', -0.5,
522 'The learning rate power.')
523
524 tf.app.flags.DEFINE_float(
525 'ftrl_initial_accumulator_value', 0.1,
526 'Starting value for the FTRL accumulators.')
527
528 tf.app.flags.DEFINE_float(
529 'ftrl_l1', 0.0, 'The FTRL l1 regularization strength.')
530
531 tf.app.flags.DEFINE_float(
532 'ftrl_l2', 0.0, 'The FTRL l2 regularization strength.')
533
534 tf.app.flags.DEFINE_float(
535 'momentum', 0.9,
536 'The momentum for the MomentumOptimizer and RMSPropOptimizer.')
537
538 tf.app.flags.DEFINE_float('rmsprop_decay', 0.9, 'Decay term for RMSProp.')
539
540 #######################
541 # Learning Rate Flags #
542 #######################
543
544 tf.app.flags.DEFINE_string(
545 'learning_rate_decay_type',
546 'exponential',
547 'Specifies how the learning rate is decayed. One of "fixed", "exponential",'
548 ' or "polynomial"')
549
550 tf.app.flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')
551
552 tf.app.flags.DEFINE_float(
553 'end_learning_rate', 0.0001,
554 'The minimal end learning rate used by a polynomial decay learning rate.')
555
556 tf.app.flags.DEFINE_float(
557 'label_smoothing', 0.0, 'The amount of label smoothing.')
558
559 tf.app.flags.DEFINE_float(
560 'learning_rate_decay_factor', 0.94, 'Learning rate decay factor.')
561
562 tf.app.flags.DEFINE_float(
563 'num_epochs_per_decay', 2.0,
564 'Number of epochs after which learning rate decays.')
565
566 tf.app.flags.DEFINE_bool(
567 'sync_replicas', False,
568 'Whether or not to synchronize the replicas during training.')
569
570 tf.app.flags.DEFINE_integer(
571 'replicas_to_aggregate', 1,
572 'The Number of gradients to collect before updating params.')
573
574 tf.app.flags.DEFINE_float(
575 'moving_average_decay', None,
576 'The decay to use for the moving average.'
577 'If left as None, then moving averages are not used.')
578
579 #######################
580 # Dataset Flags #
581 #######################
582
583 tf.app.flags.DEFINE_string(
584 'dataset_name', 'imagenet', 'The name of the dataset to load.')
585
586 tf.app.flags.DEFINE_string(
587 'dataset_split_name', 'train', 'The name of the train/test split.')
588
589 tf.app.flags.DEFINE_string(
590 'dataset_dir', None, 'The directory where the dataset files are stored.')
591
592 tf.app.flags.DEFINE_integer(
593 'labels_offset', 0,
594 'An offset for the labels in the dataset. This flag is primarily used to '
595 'evaluate the VGG and ResNet architectures which do not use a background '
596 'class for the ImageNet dataset.')
597
598 tf.app.flags.DEFINE_string(
599 'model_name', 'inception_v3', 'The name of the architecture to train.')
600
601 tf.app.flags.DEFINE_string(
602 'preprocessing_name', None, 'The name of the preprocessing to use. If left '
603 'as `None`, then the model_name flag is used.')
604
605 tf.app.flags.DEFINE_integer(
606 'batch_size', 32, 'The number of samples in each batch.')
607
608 tf.app.flags.DEFINE_integer(
609 'train_image_size', None, 'Train image size')
610
611 tf.app.flags.DEFINE_integer('max_number_of_steps', None,
612 'The maximum number of training steps.')
613
614 #####################
615 # Fine-Tuning Flags #
616 #####################
617
618 tf.app.flags.DEFINE_string(
619 'checkpoint_path', None,
620 'The path to a checkpoint from which to fine-tune.')
621
622 tf.app.flags.DEFINE_string(
623 'checkpoint_exclude_scopes', None,
624 'Comma-separated list of scopes of variables to exclude when restoring '
625 'from a checkpoint.')
626
627 tf.app.flags.DEFINE_string(
628 'trainable_scopes', None,
629 'Comma-separated list of scopes to filter the set of variables to train.'
630 'By default, None would train all the variables.')
631
632 tf.app.flags.DEFINE_boolean(
633 'ignore_missing_vars', False,
634 'When restoring a checkpoint would ignore missing variables.')
635
636 FLAGS = tf.app.flags.FLAGS
637
638
639 def _configure_learning_rate(num_samples_per_epoch, global_step):
640 """Configures the learning rate.
641
642 Args:
643 num_samples_per_epoch: The number of samples in each epoch of training.
644 global_step: The global_step tensor.
645
646 Returns:
647 A `Tensor` representing the learning rate.
648
649 Raises:
650 ValueError: if
651 """
652 decay_steps = int(num_samples_per_epoch / FLAGS.batch_size *
653 FLAGS.num_epochs_per_decay)
654 if FLAGS.sync_replicas:
655 decay_steps /= FLAGS.replicas_to_aggregate
656 #dacay,衰退
657
658
659 #下面是几种学习速率的变化形式，可以是指数型衰退，可以是固定不变，也可以使多项式型衰退。
660 if FLAGS.learning_rate_decay_type == 'exponential':
661 return tf.train.exponential_decay(FLAGS.learning_rate,
662 global_step,
663 decay_steps,
664 FLAGS.learning_rate_decay_factor,
665 staircase=True,
666 name='exponential_decay_learning_rate')
667 elif FLAGS.learning_rate_decay_type == 'fixed':
668 return tf.constant(FLAGS.learning_rate, name='fixed_learning_rate')
669 elif FLAGS.learning_rate_decay_type == 'polynomial':
670 return tf.train.polynomial_decay(FLAGS.learning_rate,
671 global_step,
672 decay_steps,
673 FLAGS.end_learning_rate,
674 power=1.0,
675 cycle=False,
676 name='polynomial_decay_learning_rate')
677 else:
678 raise ValueError('learning_rate_decay_type [%s] was not recognized',
679 FLAGS.learning_rate_decay_type)
680
681
682
683 #选择优化方法（优化器，大概是求偏导的具体数值计算方法？），tf内置了许多优化方法，类比梯度下降，细节黑箱即可。
684 def _configure_optimizer(learning_rate):
685 """Configures the optimizer used for training.
686
687 Args:
688 learning_rate: A scalar or `Tensor` learning rate.
689
690 Returns:
691 An instance of an optimizer.
692
693 Raises:
694 ValueError: if FLAGS.optimizer is not recognized.
695 """
696 if FLAGS.optimizer == 'adadelta':
697 optimizer = tf.train.AdadeltaOptimizer(
698 learning_rate,
699 rho=FLAGS.adadelta_rho,
700 epsilon=FLAGS.opt_epsilon)
701 elif FLAGS.optimizer == 'adagrad':
702 optimizer = tf.train.AdagradOptimizer(
703 learning_rate,
704 initial_accumulator_value=FLAGS.adagrad_initial_accumulator_value)
705 elif FLAGS.optimizer == 'adam':
706 optimizer = tf.train.AdamOptimizer(
707 learning_rate,
708 beta1=FLAGS.adam_beta1,
709 beta2=FLAGS.adam_beta2,
710 epsilon=FLAGS.opt_epsilon)
711 elif FLAGS.optimizer == 'ftrl':
712 optimizer = tf.train.FtrlOptimizer(
713 learning_rate,
714 learning_rate_power=FLAGS.ftrl_learning_rate_power,
715 initial_accumulator_value=FLAGS.ftrl_initial_accumulator_value,
716 l1_regularization_strength=FLAGS.ftrl_l1,
717 l2_regularization_strength=FLAGS.ftrl_l2)
718 elif FLAGS.optimizer == 'momentum':
719 optimizer = tf.train.MomentumOptimizer(
720 learning_rate,
721 momentum=FLAGS.momentum,
722 name='Momentum')
723 elif FLAGS.optimizer == 'rmsprop':
724 optimizer = tf.train.RMSPropOptimizer(
725 learning_rate,
726 decay=FLAGS.rmsprop_decay,
727 momentum=FLAGS.momentum,
728 epsilon=FLAGS.opt_epsilon)
729 elif FLAGS.optimizer == 'sgd':
730 optimizer = tf.train.GradientDescentOptimizer(learning_rate)
731 else:
732 raise ValueError('Optimizer [%s] was not recognized', FLAGS.optimizer)
733 return optimizer
734
735
736 def _get_init_fn():
737 """Returns a function run by the chief worker to warm-start the training.
738
739 Note that the init_fn is only run when initializing the model during the very
740 first global step.
741
742 Returns:
743 An init function run by the supervisor.
744 """
745 if FLAGS.checkpoint_path is None:
746 return None
747
748 # Warn the user if a checkpoint exists in the train_dir. Then we'll be
749 # ignoring the checkpoint anyway.
750 if tf.train.latest_checkpoint(FLAGS.train_dir):
751 tf.logging.info(
752 'Ignoring --checkpoint_path because a checkpoint already exists in %s'
753 % FLAGS.train_dir)
754 return None
755
756 exclusions = [] #python的list数据类型
757 if FLAGS.checkpoint_exclude_scopes:
758 exclusions = [scope.strip()
759 for scope in FLAGS.checkpoint_exclude_scopes.split(',')]
760
761 # TODO(sguada) variables.filter_variables()
762 variables_to_restore = []
763 for var in slim.get_model_variables():
764 excluded = False
765 for exclusion in exclusions:
766 if var.op.name.startswith(exclusion):
767 excluded = True
768 break
769 if not excluded:
770 variables_to_restore.append(var)
771
772 if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
773 checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
774 else:
775 checkpoint_path = FLAGS.checkpoint_path
776
777 tf.logging.info('Fine-tuning from %s' % checkpoint_path)
778
779 return slim.assign_from_checkpoint_fn(
780 checkpoint_path,
781 variables_to_restore,
782 ignore_missing_vars=FLAGS.ignore_missing_vars)
783
784
785 def _get_variables_to_train():
786 """Returns a list of variables to train.
787
788 Returns:
789 A list of variables to train by the optimizer.
790 """
791 if FLAGS.trainable_scopes is None:
792 return tf.trainable_variables()
793 else:
794 scopes = [scope.strip() for scope in FLAGS.trainable_scopes.split(',')]
795
796 variables_to_train = []
797 for scope in scopes:
798 variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope)
799 variables_to_train.extend(variables)
800 return variables_to_train
801
802
803 def main(_):
804 if not FLAGS.dataset_dir:
805 raise ValueError('You must supply the dataset directory with --dataset_dir')
806
807 tf.logging.set_verbosity(tf.logging.INFO)
808 with tf.Graph().as_default():
809 #######################
810 # Config model_deploy #
811 #######################
812 deploy_config = model_deploy.DeploymentConfig(
813 num_clones=FLAGS.num_clones,
814 clone_on_cpu=FLAGS.clone_on_cpu,
815 replica_id=FLAGS.task,
816 num_replicas=FLAGS.worker_replicas,
817 num_ps_tasks=FLAGS.num_ps_tasks)
818
819 # Create global_step
820 with tf.device(deploy_config.variables_device()):
821 global_step = slim.create_global_step()
822
823 ######################
824 # Select the dataset #
825 ######################
826 dataset = dataset_factory.get_dataset(
827 FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir)
828
829 ######################
830 # Select the network #
831 ######################
832 network_fn = nets_factory.get_network_fn(
833 FLAGS.model_name,
834 num_classes=(dataset.num_classes - FLAGS.labels_offset),
835 weight_decay=FLAGS.weight_decay,
836 is_training=True)
837
838 #####################################
839 # Select the preprocessing function #
840 #####################################
841 preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
842 image_preprocessing_fn = preprocessing_factory.get_preprocessing(
843 preprocessing_name,
844 is_training=True)
845
846 ##############################################################
847 # Create a dataset provider that loads data from the dataset #
848 ##############################################################
849 with tf.device(deploy_config.inputs_device()):
850 provider = slim.dataset_data_provider.DatasetDataProvider( #还是定义一些数据的读取方式
851 dataset,
852 num_readers=FLAGS.num_readers,
853 common_queue_capacity=20 * FLAGS.batch_size,
854 common_queue_min=10 * FLAGS.batch_size)
855 [image, label] = provider.get(['image', 'label'])
856 label -= FLAGS.labels_offset
857
858 train_image_size = FLAGS.train_image_size or network_fn.default_image_size
859
860 image = image_preprocessing_fn(image, train_image_size, train_image_size)
861
862 images, labels = tf.train.batch(
863 [image, label],
864 batch_size=FLAGS.batch_size,
865 num_threads=FLAGS.num_preprocessing_threads,
866 capacity=5 * FLAGS.batch_size)
867 labels = slim.one_hot_encoding( #one-hot是一种向量编码方式，n维向量只有为相应值的位置为1，其余都为0
868 labels, dataset.num_classes - FLAGS.labels_offset)
869 batch_queue = slim.prefetch_queue.prefetch_queue(
870 [images, labels], capacity=2 * deploy_config.num_clones)
871
872 ####################
873 # Define the model #
874 ####################
875 #通过复制多个网络来实现并行
876 def clone_fn(batch_queue):
877 """Allows data parallelism by creating multiple clones of network_fn."""
878 with tf.device(deploy_config.inputs_device()):
879 images, labels = batch_queue.dequeue() #dequeue，双端队列。。
880 logits, end_points = network_fn(images)
881
882 #############################
883 # Specify the loss function #
884 #############################
885 if 'AuxLogits' in end_points:
886 tf.losses.softmax_cross_entropy( #softmax函数对应使用的cost function(loss function)
887 #是corss_entropy,也就是交叉熵
888 logits=end_points['AuxLogits'], onehot_labels=labels,
889 label_smoothing=FLAGS.label_smoothing, weights=0.4, scope='aux_loss')
890 tf.losses.softmax_cross_entropy(
891 logits=logits, onehot_labels=labels,
892 label_smoothing=FLAGS.label_smoothing, weights=1.0)
893 #Label Smoothing Regularization，一种防止overfit的优化方法
894 return end_points
895
896 # Gather initial summaries.
897 summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
898
899 clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue])
900 first_clone_scope = deploy_config.clone_scope(0)
901 # Gather update_ops from the first clone. These contain, for example,
902 # the updates for the batch_norm variables created by network_fn.
903 update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope)
904
905 # Add summaries for end_points.
906 end_points = clones[0].outputs
907 for end_point in end_points:
908 x = end_points[end_point]
909 summaries.add(tf.summary.histogram('activations/' + end_point, x))
910 summaries.add(tf.summary.scalar('sparsity/' + end_point,
911 tf.nn.zero_fraction(x)))
912
913 # Add summaries for losses.
914 for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
915 summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss))
916
917 # Add summaries for variables.
918 for variable in slim.get_model_variables():
919 summaries.add(tf.summary.histogram(variable.op.name, variable))
920 #突然画图
921
922 #################################
923 # Configure the moving averages # #参考moving averages的wiki
924 #################################
925 if FLAGS.moving_average_decay:
926 moving_average_variables = slim.get_model_variables()
927 variable_averages = tf.train.ExponentialMovingAverage(
928 FLAGS.moving_average_decay, global_step)
929 else:
930 moving_average_variables, variable_averages = None, None
931
932 #########################################
933 # Configure the optimization procedure. #
934 #########################################
935 with tf.device(deploy_config.optimizer_device()):
936 learning_rate = _configure_learning_rate(dataset.num_samples, global_step)
937 optimizer = _configure_optimizer(learning_rate)
938 summaries.add(tf.summary.scalar('learning_rate', learning_rate))
939
940
941 #在分布式系统上训练的同步...
942 if FLAGS.sync_replicas:
943 # If sync_replicas is enabled, the averaging will be done in the chief
944 # queue runner.
945 optimizer = tf.train.SyncReplicasOptimizer(
946 opt=optimizer,
947 replicas_to_aggregate=FLAGS.replicas_to_aggregate,
948 variable_averages=variable_averages,
949 variables_to_average=moving_average_variables,
950 replica_id=tf.constant(FLAGS.task, tf.int32, shape=()),
951 total_num_replicas=FLAGS.worker_replicas)
952 elif FLAGS.moving_average_decay:
953 # Update ops executed locally by trainer.
954 update_ops.append(variable_averages.apply(moving_average_variables))
955
956 # Variables to train.
957 variables_to_train = _get_variables_to_train()
958
959 # and returns a train_tensor and summary_op
960 total_loss, clones_gradients = model_deploy.optimize_clones(
961 clones,
962 optimizer,
963 var_list=variables_to_train)
964 # Add total_loss to summary.
965 summaries.add(tf.summary.scalar('total_loss', total_loss))
966
967 # Create gradient updates.
968 grad_updates = optimizer.apply_gradients(clones_gradients,
969 global_step=global_step)
970 update_ops.append(grad_updates)
971
972 update_op = tf.group(*update_ops)
973 with tf.control_dependencies([update_op]):
974 train_tensor = tf.identity(total_loss, name='train_op')
975
976 # Add the summaries from the first clone. These contain the summaries
977 # created by model_fn and either optimize_clones() or _gather_clone_loss().
978 summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
979 first_clone_scope))
980
981 # Merge all summaries together.
982 summary_op = tf.summary.merge(list(summaries), name='summary_op')
983
984
985 ###########################
986 # Kicks off the training. #
987 ###########################
988 slim.learning.train(
989 train_tensor,
990 logdir=FLAGS.train_dir,
991 master=FLAGS.master,
992 is_chief=(FLAGS.task == 0),
993 init_fn=_get_init_fn(),
994 summary_op=summary_op,
995 number_of_steps=FLAGS.max_number_of_steps,
996 log_every_n_steps=FLAGS.log_every_n_steps,
997 save_summaries_secs=FLAGS.save_summaries_secs,
998 save_interval_secs=FLAGS.save_interval_secs,
999 sync_optimizer=optimizer if FLAGS.sync_replicas else None)
1000
1001
1002 if __name__ == '__main__':
1003 tf.app.run()