Tensorflow Object Detection (사용자 정의 이미지)

------------------폴더구조 ------------------------------------------ + annotations: contains the xml files in PASCAL VOC format + data: contains the input file for the TF object detection API and the label files (csv) + images: contains the image data in jpg format + training: contains the pipeline configuration file, frozen model and labelmap - a few handy scripts: generate_tfrecord.py is used to generate the input files for the TF API and xml_to_csv.py is used to convert the xml files into one csv - a few jupyter notebooks: draw boxes is used to plot some of the data and split labels is used to split the full labels into train and test labels ---------------------------------------------------------------------------------- 1. Tensorflow가 인식하는 데이타 format인 " TFRecord file format"으로 만들어준다. 1) 일단 detecting 할 이미지를 images하위의 train, test 폴더에 모은다. (train 이미지는 최소 100개이상, test이미지는 40개 정도 마련) 2) labelImg 프로그램(https://github.com/tzutalin/labelImg)을 이용하여 모아놓은 이미지에서 영역을 선택 후 라벨링(예. bag, pants....등등)하고 annotation폴더에 저장한다. (labelImg 는 PyQt4가 사전에 설치되어있어야 함. https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyqt4 에서 자신에 맞는 버전을 다운로드 한 후 pip install "다운로드한 whl파일"하면 됨) ==> 라벨링하고 저장하면 이미지명으로 된 xml파일이 생긴다. ----------------------------생성된 XML파일 예시----------------------------

<annotation>
<folder>Backpack</folder>
<filename>1709042528_LS1.jpg</filename>
<path>D:\WORK\tensor_test\mydata\images\Backpack\1709042528_LS1.jpg</path>
<source>
<database>Unknown</database>
</source>
<size>
<width>300</width>
<height>450</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>backpack</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>58</xmin>
<ymin>107</ymin>
<xmax>239</xmax>
<ymax>380</ymax>
</bndbox>
</object>
</annotation>

---------------------------------------------------------------------------- 3) 생성된 XML 파일을 xml_to_csv.py를 이용하여 CSV파일로 변환한다. (참조소스 : https://github.com/datitran/raccoon_dataset) images 폴더 하위의 train, test 폴더의 xml파일을 검색한다. - 명령어 : python xml_to_csv.py annotation(위에 xml이 모여있는 폴더명) - 명령을 실행하고 나면 data 폴더 하위에 train_labels.csv, test_labels.csv 파일이 생성된다. --------------------------xml_to_csv.py 소스------------------------------------------------------ import os
import glob
import pandas as pd
import xml.etree.ElementTree as ET

def xml_to_csv(path):
xml_list = []
for xml_file in glob.glob(path + '/*.xml'):
tree = ET.parse(xml_file)
root = tree.getroot()
for member in root.findall('object'):
value = (root.find('filename').text,
int(root.find('size')[0].text),
int(root.find('size')[1].text),
member[0].text,
int(member[4][0].text),
int(member[4][1].text),
int(member[4][2].text),
int(member[4][3].text)
)
xml_list.append(value)
column_name = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']
xml_df = pd.DataFrame(xml_list, columns=column_name)
return xml_df


def main():
for directory in ['train','test']:
image_path = os.path.join(os.getcwd(), 'images/{}'.format(directory))
xml_df = xml_to_csv(image_path)
xml_df.to_csv('data/{}_labels.csv'.format(directory), index=None)
print('Successfully converted xml to csv.')


main()
---------------------------------------------------------------------------------------------------------------- 4) 변환된 CSV 파일을 generate_tfrecord.py를 이용하여 TFRecord files(확장자 : .record)로 변환한다. (train, test 이미지 각각 실행) - 먼저 소스의 row_label을 label_map.pbtxt 의 라벨명별로 수정해준다. - train 용 명령어 : python generate_tfrecord.py --csv_input=data/train_labels.csv --output_path=train.record - test 용 명령어 : python generate_tfrecord.py --csv_input=data/test_labels.csv --output_path=test.record - 위 명령어 실행시 data 폴더 하위에 train.record와 test.record 가 생성된다. --------------------- generate_tfrecord.py 소스 ------------------------------------------------------------------ """
Usage:
# From tensorflow/models/
# Create train data:
python generate_tfrecord.py --csv_input=data/train_labels.csv --output_path=train.record

# Create test data:
python generate_tfrecord.py --csv_input=data/test_labels.csv --output_path=test.record
"""
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import os
import io
import pandas as pd
import tensorflow as tf

from PIL import Image
from object_detection.utils import dataset_util
from collections import namedtuple, OrderedDict

flags = tf.app.flags
flags.DEFINE_string('csv_input', '', 'Path to the CSV input')
flags.DEFINE_string('output_path', '', 'Path to output TFRecord')
FLAGS = flags.FLAGS

# TO-DO replace this with label map
def class_text_to_int(row_label):
if row_label == 'backpack':
return 1
else:
None

def split(df, group):
data = namedtuple('data', ['filename', 'object'])
gb = df.groupby(group)
return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]


def create_tf_example(group, path):
with tf.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid:
encoded_jpg = fid.read()
encoded_jpg_io = io.BytesIO(encoded_jpg)
image = Image.open(encoded_jpg_io)
width, height = image.size

filename = group.filename.encode('utf8')
image_format = b'jpg'
xmins = []
xmaxs = []
ymins = []
ymaxs = []
classes_text = []
classes = []

for index, row in group.object.iterrows():
xmins.append(row['xmin'] / width)
xmaxs.append(row['xmax'] / width)
ymins.append(row['ymin'] / height)
ymaxs.append(row['ymax'] / height)
classes_text.append(row['class'].encode('utf8'))
classes.append(class_text_to_int(row['class']))

tf_example = tf.train.Example(features=tf.train.Features(feature={
'image/height': dataset_util.int64_feature(height),
'image/width': dataset_util.int64_feature(width),
'image/filename': dataset_util.bytes_feature(filename),
'image/source_id': dataset_util.bytes_feature(filename),
'image/encoded': dataset_util.bytes_feature(encoded_jpg),
'image/format': dataset_util.bytes_feature(image_format),
'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),
'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),
'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
'image/object/class/label': dataset_util.int64_list_feature(classes),
}))
return tf_example


def main(_):
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
path = os.path.join(os.getcwd(), 'images/test')
print("path>>>",path)
examples = pd.read_csv(FLAGS.csv_input)
grouped = split(examples, 'filename')
for group in grouped:
tf_example = create_tf_example(group, path)
writer.write(tf_example.SerializeToString())

writer.close()
output_path = os.path.join(os.getcwd(), FLAGS.output_path)
print('Successfully created the TFRecords: {}'.format(output_path))

if __name__ == '__main__':
tf.app.run()
---------------------------------------- generate_tfrecord.py end ----------------------------------- 2. 학습시키기 전 참고모델 및 셋팅 1) 먼저 checkpoint 모델 파일은 위 폴더구조의 최상위 폴더에, config파일은 training폴더에 다운 받는다. wget https://raw.githubusercontent.com/tensorflow/models/master/object_detection/samples/configs/ssd_mobilenet_v1_pets.config <--configuration wget https://storage.googleapis.com/download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_coco_11_06_2017.tar.gz <-- 모델안에 모델 및 checkpoint 파일이 있다. * 다운받은 configuration 파일 수정방법 - 클래스수만큼 num_classes를 수를 설정 (예. class가 가방 하나이면 1 셋팅) - batch size는 메모리 여건에 따라서 조정 - fine_tune_checkpoint 경로 변경 - 학습 횟수가 디폴트로 20만번으로 되어있다. 상황에 따라서 num_steps을 조정 - train_input_reader의 input_path, label_map_path 경로 변경 - eval_input_readerinput_path, label_map_path 경로 변경 - TODO 기타 설정 값들에 대한 정리 자료 작성 필요 아래 config 는 다운받은 모델에 맞는 config를 다운받는다. https://github.com/tensorflow/models/tree/master/research/object_detection/samples/configs ----------------------- ssd_mobilenet_v1_[class들 명칭. 예) bag.config 예시 ------------------------------------

# SSD with Mobilenet v1, configured for the mac-n-cheese dataset.
# Users should configure the fine_tune_checkpoint field in the train config as
# well as the label_map_path and input_path fields in the train_input_reader and
# eval_input_reader. Search for "${YOUR_GCS_BUCKET}" to find the fields that
# should be configured. model {
ssd {
num_classes: 1 # 수정할 부분(클레스 수)
box_coder {
faster_rcnn_box_coder {
y_scale: 10.0
x_scale: 10.0
height_scale: 5.0
width_scale: 5.0
}
}
matcher {
argmax_matcher {
matched_threshold: 0.5
unmatched_threshold: 0.5
ignore_thresholds: false
negatives_lower_than_unmatched: true
force_match_for_each_row: true
}
}
similarity_calculator {
iou_similarity {
}
}
anchor_generator {
ssd_anchor_generator {
num_layers: 6
min_scale: 0.2
max_scale: 0.95
aspect_ratios: 1.0
aspect_ratios: 2.0
aspect_ratios: 0.5
aspect_ratios: 3.0
aspect_ratios: 0.3333
}
}
image_resizer {
fixed_shape_resizer {
height: 300
width: 300
}
}
box_predictor {
convolutional_box_predictor {
min_depth: 0
max_depth: 0
num_layers_before_predictor: 0
use_dropout: false
dropout_keep_probability: 0.8
kernel_size: 1
box_code_size: 4
apply_sigmoid_to_scores: false
conv_hyperparams {
activation: RELU_6,
regularizer {
l2_regularizer {
weight: 0.00004
}
}
initializer {
truncated_normal_initializer {
stddev: 0.03
mean: 0.0
}
}
batch_norm {
train: true,
scale: true,
center: true,
decay: 0.9997,
epsilon: 0.001,
}
}
}
}
feature_extractor {
type: 'ssd_mobilenet_v1'
min_depth: 16
depth_multiplier: 1.0
conv_hyperparams {
activation: RELU_6,
regularizer {
l2_regularizer {
weight: 0.00004
}
}
initializer {
truncated_normal_initializer {
stddev: 0.03
mean: 0.0
}
}
batch_norm {
train: true,
scale: true,
center: true,
decay: 0.9997,
epsilon: 0.001,
}
}
}
loss {
classification_loss {
weighted_sigmoid {
anchorwise_output: true
}
}
localization_loss {
weighted_smooth_l1 {
anchorwise_output: true
}
}
hard_example_miner {
num_hard_examples: 3000
iou_threshold: 0.99
loss_type: CLASSIFICATION
max_negatives_per_positive: 3
min_negatives_per_image: 0
}
classification_weight: 1.0
localization_weight: 1.0
}
normalize_loss_by_num_matches: true
post_processing {
batch_non_max_suppression {
score_threshold: 1e-8
iou_threshold: 0.6
max_detections_per_class: 100
max_total_detections: 100
}
score_converter: SIGMOID
}
}
} train_config: {
batch_size: 8 # 배치사이즈가 너무 크면 메모리 에러난다. OOP (Out of memory)
optimizer {
rms_prop_optimizer: {
learning_rate: {
exponential_decay_learning_rate {
initial_learning_rate: 0.004
decay_steps: 800720
decay_factor: 0.95
}
}
momentum_optimizer_value: 0.9
decay: 0.9
epsilon: 1.0
}
}
fine_tune_checkpoint: "faster_rcnn_resnet101_coco_11_06_2017/model.ckpt" # 수정할 부분
from_detection_checkpoint: true

num_steps: 100 #실행할 step 수 셋팅
data_augmentation_options {
random_horizontal_flip {
}
}
data_augmentation_options {
ssd_random_crop {
}
}
} train_input_reader: {
tf_record_input_reader {
input_path: "data/train.record" # 수정할 부분 (앞단계에서 생성함)
}
label_map_path: "data/object-detection.pbtxt" # 수정할 부분(아래설명과 같이서 생성필요)
} eval_config: {
num_examples: 40 # 예제샘플 수
} eval_input_reader: {
tf_record_input_reader {
input_path: "data/test.record" # 수정할 부분(앞단계에서 생성함)
}
label_map_path: "training/object-detection.pbtxt" # 수정할 부분
shuffle: false
num_readers: 1
}

---------------------------------------------------------------------------------------------------------------------------- 2) 위에서 라벨링 한 내용을 json형태의 특정이름_label_map.pbtxt (위 소스에서 사용한 object-detection.pbtxt 로 해도 됨.) 파일을 data 및 training 폴더 하위에 만들어준다. ------------------- label_map.pbtxt 파일 예제 --------------------------------------------------- item { id: 1 name: 'Abyssinian' } item { id: 2 name: 'american_bulldog' } ------------------------------------------------------------------------------------------------- 3. 이제 모든 준비가 끝나서 학습 시킨다. models/object_detection (이건 tensorflow의 object detection api 를 다운로드 받으면 됨.) 하위의 train.py 를 실행시킨다. 명령어 : python train.py --logtostderr --train_dir=training/ --pipeline_config_path=training/ssd_mobilenet_v1_pets.config - GPU 환경하에 돌리기 위해 아래설정을 해준다. 1) 환경설정에 CUDA_HOME 설정 : D:\ML_Program\CUDA Toolkit\v9.0 2) path에 추가 - D:\ML_Program\CUDA Toolkit\v9.0\extras\CUPTI\libx64 - D:\ML_Program\CUDA Toolkit\v9.0\lib\x64 * 실행시 cannot import name 'preprocessor_pb2' 에러가 발생하면 1) proto 확장자를 컴파일 하기 위한 실행을 위한 파일(https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-win32.zip)을 다운받은 후 압축을 푼다. (아무데나) 2) 위에서 압축을 풀면 /bin 폴더 하위에 있는 protoc.exe가 있는데 그것을 google 모델 하위의 research\ 로 복사한다. 3) 설명에는 protoc.exe ./object_detection/protos/*.proto --python_out=. 로 명령어가 되어있으나 "*"(Asterisk) 가 안되어 "No such file or directory" 에러가 난다. 그래서 loop를 돌릴 수 있는 배치파일을 하나 만든다. 이름은 protoloop.bat 로 research\ 하위에 만든다. --------------------------protoloop.bat 내용--------------------------- for /f %%v in ('dir object_detection\protos\*.proto /b') do ( protoc object_detection/protos/%%v --python_out=. ) -------------------------------------------------------------------------- 위 배치파일을 실행시키면 .proto 파일이 .py (python파일)로 생성된다. 4) 추가로 D:\ML\models\research\slim 을 path설정한다. 이유는 slim 하위경로의 deployment 를 trainer.py 에서 사용한다. pyCharm 에서는 run>Configuration에서 윈도우 환경설정의 PATH 키값 "PYTHONPATH" 를 가져오므로 윈도우 환경설정에 PYTHONPATH 키를 새로 생성하여 D:\ML\models\research\slim 를 입력한다. ----아래와 같은 로그가 나옴 ----------------------------------------------- Barring errors, you should see output like:
INFO:tensorflow:global step 11788: loss = 0.6717 (0.398 sec/step)
INFO:tensorflow:global step 11789: loss = 0.5310 (0.436 sec/step)
INFO:tensorflow:global step 11790: loss = 0.6614 (0.405 sec/step)
INFO:tensorflow:global step 11791: loss = 0.7758 (0.460 sec/step)
INFO:tensorflow:global step 11792: loss = 0.7164 (0.378 sec/step)
INFO:tensorflow:global step 11793: loss = 0.8096 (0.393 sec/step)
---------------------------------------------------------------------------- 다 돌아가면 training 폴더 안에 pipeline.config 가 생김. 4. tensorboard 를 실행시킨다. (training 폴더 지정) : loss 확인 명력어 : tensorboard --logdir=./training  5. Tensorflow Graph 파일을 내보내기한다.  python export_inference_graph.py \ --input_type image_tensor \  --pipeline_config_path TELCO_OBJ_DETECTION/training/ssd_mobilenet_v1_pets.config \ <-- 이전에 사용했던 config 경로 셋팅 --trained_checkpoint_prefix TELCO_OBJ_DETECTION/training/model.ckpt-33354 \ <-- 학습된 체크아웃 파일 지정 --output_directory output_inference_graph.pb <-- output_inference_graph.pb 로 이름을 셋팅 ex)  object_detection 폴더하위에 있다.       python export_inference_graph.py --input_type image_tensor  --pipeline_config_path ../training/ssd_mobilenet_v1_coco.config --trained_checkpoint_prefix ../training/model.ckpt-100 --output_directory ../output_inference_graph.pb 6. 실제 임의의 이미지로 실시간 테스트 명령어 : python objdetection.py –config=opt  ------------------------------------ objectdetect.py ------------------------------------------------------- import numpy as np
import os
import six.moves.urllib as urllib
import sys
import tarfile
import tensorflow as tf
import zipfile

from collections import defaultdict
from io import StringIO
#from matplotlib import pyplot as plt
#import matplotlib.pyplot as plt
from PIL import Image
import object_detection.utils.label_map_util as label_map_util
import object_detection.utils.visualization_utils as vis_util
import cv2


# This is needed since the notebook is stored in the object_detection folder.
#sys.path.append("..")
import object_detection.utils.ops as utils_ops

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

#if tf.__version__ < '1.4.0':
# raise ImportError('Please upgrade your tensorflow installation to v1.4.* or later!')
# This is needed to display the images.
#%matplotlib inline



# What model to download.
MODEL_NAME = 'ssd_mobilenet_v1_coco_2017_11_17'
MODEL_FILE = MODEL_NAME + '.tar.gz'
DOWNLOAD_BASE = 'https://download.tensorflow.org/models/object_detection/'

# Path to frozen detection graph. This is the actual model that is used for the object detection.
PATH_TO_CKPT = MODEL_NAME + '/frozen_inference_graph.pb'

# List of the strings that is used to add correct label for each box.
PATH_TO_LABELS = os.path.join('object_detection\data', 'mscoco_label_map.pbtxt')

NUM_CLASSES = 90

opener = urllib.request.URLopener()
opener.retrieve(DOWNLOAD_BASE + MODEL_FILE, MODEL_FILE)
tar_file = tarfile.open(MODEL_FILE)
for file in tar_file.getmembers():
file_name = os.path.basename(file.name)
if 'frozen_inference_graph.pb' in file_name:
tar_file.extract(file, os.getcwd())

detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
print(">>>>PATH_TO_LABELS>>>>>>>>>>",PATH_TO_LABELS)
label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
category_index = label_map_util.create_category_index(categories)

def load_image_into_numpy_array(image):
(im_width, im_height) = image.size
return np.array(image.getdata()).reshape(
(im_height, im_width, 3)).astype(np.uint8)

# For the sake of simplicity we will use only 2 images:
# image1.jpg
# image2.jpg
# If you want to test the code with your images, just add path to the images to the TEST_IMAGE_PATHS.
PATH_TO_TEST_IMAGES_DIR = 'object_detection\\test_images'
TEST_IMAGE_PATHS = [ os.path.join(PATH_TO_TEST_IMAGES_DIR, 'image{}.jpg'.format(i)) for i in range(1, 4) ]

# Size, in inches, of the output images.
IMAGE_SIZE = (12, 4)

def run_inference_for_single_image(image, graph):
with graph.as_default():
with tf.Session() as sess:
# Get handles to input and output tensors
ops = tf.get_default_graph().get_operations()
all_tensor_names = {output.name for op in ops for output in op.outputs}
tensor_dict = {}
for key in [
'num_detections', 'detection_boxes', 'detection_scores',
'detection_classes', 'detection_masks'
]:
tensor_name = key + ':0'
if tensor_name in all_tensor_names:
tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(
tensor_name)
if 'detection_masks' in tensor_dict:
# The following processing is only for single image
detection_boxes = tf.squeeze(tensor_dict['detection_boxes'], [0])
detection_masks = tf.squeeze(tensor_dict['detection_masks'], [0])
# Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.
real_num_detection = tf.cast(tensor_dict['num_detections'][0], tf.int32)
detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1])
detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1])
detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(
detection_masks, detection_boxes, image.shape[0], image.shape[1])
detection_masks_reframed = tf.cast(
tf.greater(detection_masks_reframed, 0.5), tf.uint8)
# Follow the convention by adding back the batch dimension
tensor_dict['detection_masks'] = tf.expand_dims(
detection_masks_reframed, 0)
image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0')

# Run inference
output_dict = sess.run(tensor_dict,
feed_dict={image_tensor: np.expand_dims(image, 0)})

# all outputs are float32 numpy arrays, so convert types as appropriate
output_dict['num_detections'] = int(output_dict['num_detections'][0])
output_dict['detection_classes'] = output_dict[
'detection_classes'][0].astype(np.uint8)
output_dict['detection_boxes'] = output_dict['detection_boxes'][0]
output_dict['detection_scores'] = output_dict['detection_scores'][0]
if 'detection_masks' in output_dict:
output_dict['detection_masks'] = output_dict['detection_masks'][0]
return output_dict

m=0
for image_path in TEST_IMAGE_PATHS:
m=m+1
print(">>>>>>>image_path>>>>>>>>",image_path)
image = Image.open(image_path)
# the array based representation of the image will be used later in order to prepare the
# result image with boxes and labels on it.
image_np = load_image_into_numpy_array(image)
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
# Actual detection.
output_dict = run_inference_for_single_image(image_np, detection_graph)
# Visualization of the results of a detection.
vis_util.visualize_boxes_and_labels_on_image_array(
image_np,
output_dict['detection_boxes'],
output_dict['detection_classes'],
output_dict['detection_scores'],
category_index,
instance_masks=output_dict.get('detection_masks'),
use_normalized_coordinates=True,
line_thickness=8)
#plt.figure(figsize=IMAGE_SIZE)
#plt.imshow(image_np)

cv2.imshow('img'+str(m),image_np)

cv2.waitKey(0)
-----------------------------------------------------------------------------------------------------------------