_archiving/contribution/daekeun-ml/tensorflow-in-sagemaker-workshop/training_script/cifar10_keras_pipe_solution.py [133:308]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - dataset = dataset.repeat(epochs) dataset = dataset.prefetch(10) # Parse records. dataset = dataset.map( _dataset_parser, num_parallel_calls=10) # Potentially shuffle records. if channel_name == 'train': # Ensure that the capacity is sufficiently large to provide good random # shuffling. buffer_size = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 0.4) + 3 * batch_size dataset = dataset.shuffle(buffer_size=buffer_size) # Batch it up. dataset = dataset.batch(batch_size, drop_remainder=True) iterator = dataset.make_one_shot_iterator() image_batch, label_batch = iterator.get_next() return {INPUT_TENSOR_NAME: image_batch}, label_batch def _train_preprocess_fn(image): """Preprocess a single training image of layout [height, width, depth].""" # Resize the image to add four extra pixels on each side. image = tf.image.resize_image_with_crop_or_pad(image, HEIGHT + 8, WIDTH + 8) # Randomly crop a [HEIGHT, WIDTH] section of the image. image = tf.random_crop(image, [HEIGHT, WIDTH, DEPTH]) # Randomly flip the image horizontally. image = tf.image.random_flip_left_right(image) return image def _dataset_parser(value): """Parse a CIFAR-10 record from value.""" featdef = { 'image': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), } example = tf.parse_single_example(value, featdef) image = tf.decode_raw(example['image'], tf.uint8) image.set_shape([DEPTH * HEIGHT * WIDTH]) # Reshape from [depth * height * width] to [depth, height, width]. image = tf.cast( tf.transpose(tf.reshape(image, [DEPTH, HEIGHT, WIDTH]), [1, 2, 0]), tf.float32) label = tf.cast(example['label'], tf.int32) image = _train_preprocess_fn(image) return image, tf.one_hot(label, NUM_CLASSES) def save_model(model, output): signature = tf.saved_model.signature_def_utils.predict_signature_def( inputs={'inputs': model.input}, outputs={'scores': model.output}) builder = tf.saved_model.builder.SavedModelBuilder(output+'/1/') builder.add_meta_graph_and_variables( sess=K.get_session(), tags=[tf.saved_model.tag_constants.SERVING], signature_def_map={"serving_default": signature}) builder.save() logging.info("Model successfully saved at: {}".format(output)) return def main(args): logging.info("getting data") train_dataset = train_input_fn() eval_dataset = eval_input_fn() validation_dataset = validation_input_fn() logging.info("configuring model") model = keras_model_fn(args.learning_rate, args.weight_decay, args.optimizer, args.momentum) callbacks = [] # ----- 수정 부분 (경로 수정) ----- callbacks.append(ModelCheckpoint(args.model_output_dir + '/checkpoint-{epoch}.h5')) logging.info("Starting training") model.fit(x=train_dataset[0], y=train_dataset[1], steps_per_epoch=(num_examples_per_epoch('train') // args.batch_size), epochs=args.epochs, validation_data=validation_dataset, validation_steps=(num_examples_per_epoch('validation') // args.batch_size), callbacks=callbacks) score = model.evaluate(eval_dataset[0], eval_dataset[1], steps=num_examples_per_epoch('eval') // args.batch_size, verbose=0) logging.info('Test loss:{}'.format(score[0])) logging.info('Test accuracy:{}'.format(score[1])) # ----- 수정 부분 (경로 수정) ----- return save_model(model, args.model_output_dir) def num_examples_per_epoch(subset='train'): if subset == 'train': return 40000 elif subset == 'validation': return 10000 elif subset == 'eval': return 10000 else: raise ValueError('Invalid data subset "%s"' % subset) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( '--train', type=str, required=False, default=os.environ['SM_CHANNEL_TRAIN'], # ----- 수정 부분 (default 경로 수정) ----- help='The directory where the CIFAR-10 input data is stored.') parser.add_argument( '--validation', type=str, required=False, default=os.environ['SM_CHANNEL_VALIDATION'], # ----- 수정 부분 (default 경로 수정) ----- help='The directory where the CIFAR-10 input data is stored.') parser.add_argument( '--eval', type=str, required=False, default=os.environ['SM_CHANNEL_EVAL'], # ----- 수정 부분 (default 경로 수정) ----- help='The directory where the CIFAR-10 input data is stored.') # ----- 수정 부분 (argument 추가) ----- parser.add_argument( '--model_output_dir', type=str, default=os.environ.get('SM_MODEL_DIR')) parser.add_argument( '--model_dir', type=str, required=True, help='The directory where the model will be stored.') parser.add_argument( '--weight-decay', type=float, default=2e-4, help='Weight decay for convolutions.') parser.add_argument( '--learning-rate', type=float, default=0.001, help="""\ This is the inital learning rate value. The learning rate will decrease during training. For more details check the model_fn implementation in this file.\ """) parser.add_argument( '--epochs', type=int, default=10, help='The number of steps to use for training.') parser.add_argument( '--batch-size', type=int, default=128, help='Batch size for training.') parser.add_argument( '--optimizer', type=str, default='adam') parser.add_argument( '--momentum', type=float, default='0.9') args = parser.parse_args() main(args) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - _archiving/contribution/daekeun-ml/tensorflow-in-sagemaker-workshop/training_script/cifar10_keras_sm_solution.py [128:303]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - dataset = dataset.repeat(epochs) dataset = dataset.prefetch(10) # Parse records. dataset = dataset.map( _dataset_parser, num_parallel_calls=10) # Potentially shuffle records. if channel_name == 'train': # Ensure that the capacity is sufficiently large to provide good random # shuffling. buffer_size = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 0.4) + 3 * batch_size dataset = dataset.shuffle(buffer_size=buffer_size) # Batch it up. dataset = dataset.batch(batch_size, drop_remainder=True) iterator = dataset.make_one_shot_iterator() image_batch, label_batch = iterator.get_next() return {INPUT_TENSOR_NAME: image_batch}, label_batch def _train_preprocess_fn(image): """Preprocess a single training image of layout [height, width, depth].""" # Resize the image to add four extra pixels on each side. image = tf.image.resize_image_with_crop_or_pad(image, HEIGHT + 8, WIDTH + 8) # Randomly crop a [HEIGHT, WIDTH] section of the image. image = tf.random_crop(image, [HEIGHT, WIDTH, DEPTH]) # Randomly flip the image horizontally. image = tf.image.random_flip_left_right(image) return image def _dataset_parser(value): """Parse a CIFAR-10 record from value.""" featdef = { 'image': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), } example = tf.parse_single_example(value, featdef) image = tf.decode_raw(example['image'], tf.uint8) image.set_shape([DEPTH * HEIGHT * WIDTH]) # Reshape from [depth * height * width] to [depth, height, width]. image = tf.cast( tf.transpose(tf.reshape(image, [DEPTH, HEIGHT, WIDTH]), [1, 2, 0]), tf.float32) label = tf.cast(example['label'], tf.int32) image = _train_preprocess_fn(image) return image, tf.one_hot(label, NUM_CLASSES) def save_model(model, output): signature = tf.saved_model.signature_def_utils.predict_signature_def( inputs={'inputs': model.input}, outputs={'scores': model.output}) builder = tf.saved_model.builder.SavedModelBuilder(output+'/1/') builder.add_meta_graph_and_variables( sess=K.get_session(), tags=[tf.saved_model.tag_constants.SERVING], signature_def_map={"serving_default": signature}) builder.save() logging.info("Model successfully saved at: {}".format(output)) return def main(args): logging.info("getting data") train_dataset = train_input_fn() eval_dataset = eval_input_fn() validation_dataset = validation_input_fn() logging.info("configuring model") model = keras_model_fn(args.learning_rate, args.weight_decay, args.optimizer, args.momentum) callbacks = [] # ----- 수정 부분 (경로 수정) ----- callbacks.append(ModelCheckpoint(args.model_output_dir + '/checkpoint-{epoch}.h5')) logging.info("Starting training") model.fit(x=train_dataset[0], y=train_dataset[1], steps_per_epoch=(num_examples_per_epoch('train') // args.batch_size), epochs=args.epochs, validation_data=validation_dataset, validation_steps=(num_examples_per_epoch('validation') // args.batch_size), callbacks=callbacks) score = model.evaluate(eval_dataset[0], eval_dataset[1], steps=num_examples_per_epoch('eval') // args.batch_size, verbose=0) logging.info('Test loss:{}'.format(score[0])) logging.info('Test accuracy:{}'.format(score[1])) # ----- 수정 부분 (경로 수정) ----- return save_model(model, args.model_output_dir) def num_examples_per_epoch(subset='train'): if subset == 'train': return 40000 elif subset == 'validation': return 10000 elif subset == 'eval': return 10000 else: raise ValueError('Invalid data subset "%s"' % subset) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( '--train', type=str, required=False, default=os.environ['SM_CHANNEL_TRAIN'], # ----- 수정 부분 (default 경로 수정) ----- help='The directory where the CIFAR-10 input data is stored.') parser.add_argument( '--validation', type=str, required=False, default=os.environ['SM_CHANNEL_VALIDATION'], # ----- 수정 부분 (default 경로 수정) ----- help='The directory where the CIFAR-10 input data is stored.') parser.add_argument( '--eval', type=str, required=False, default=os.environ['SM_CHANNEL_EVAL'], # ----- 수정 부분 (default 경로 수정) ----- help='The directory where the CIFAR-10 input data is stored.') # ----- 수정 부분 (argument 추가) ----- parser.add_argument( '--model_output_dir', type=str, default=os.environ.get('SM_MODEL_DIR')) parser.add_argument( '--model_dir', type=str, required=True, help='The directory where the model will be stored.') parser.add_argument( '--weight-decay', type=float, default=2e-4, help='Weight decay for convolutions.') parser.add_argument( '--learning-rate', type=float, default=0.001, help="""\ This is the inital learning rate value. The learning rate will decrease during training. For more details check the model_fn implementation in this file.\ """) parser.add_argument( '--epochs', type=int, default=10, help='The number of steps to use for training.') parser.add_argument( '--batch-size', type=int, default=128, help='Batch size for training.') parser.add_argument( '--optimizer', type=str, default='adam') parser.add_argument( '--momentum', type=float, default='0.9') args = parser.parse_args() main(args) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -