[Synthetic-data-with-text-and-image]
0 引言
本文是之前为了解决如何将文字贴到图片上而编写的代码,默认是如发票一类的，所以并未考虑透视变换等。且采用的是pygame粘贴方式，之前也尝试过opencv的seamlessClone粘贴。
值得注意的是，通过修改参数，增加各种干扰操作（羽化，噪音等等），生成的数据集看似丰富，其实因为内在的数据分布还是十分单一，也就是用该数据集去作为ocr的模型训练集，得到的模型仍然无法在现实生活场景上使用。因为在现实世界中，你光照角度，拍摄角度，打印机用墨等等都是一种变量，而这些变量都会让现实世界的票据呈现的文字内在数据分布是十分丰富的。而通过简单的代码生成的数据分布却并不能覆盖，或者说重叠其中一部分。故而，通过代码生成数据集的方式是无法解决ocr现实数据集不够的问题的。
所需要的操作：
1 - 下载colors_new.cp；
2 - 将下面两份代码存成对应的get_color.py 和pygame_main.py；
3 - python 运行pygame_main.py即可。
# get_color.py
import cv2
import pickle as cp
import numpy as np
class ColorSample(object):
def __init__(self):
'''colors_new.cp来自https://github.com/JarveeLee/SynthText_Chinese_version/tree/master/data/models/colors_new.cp '''
with open('colors_new.cp','rb') as f:
self.colorsRGB = cp.load(f,encoding='latin-1')
self.ncol = self.colorsRGB.shape[0]#4941
# convert color-means from RGB to LAB for better nearest neighbour
# computations:
self.colorsLAB = np.r_[self.colorsRGB[:,0:3], self.colorsRGB[:,6:9]].astype('uint8')
self.colorsLAB = np.squeeze(cv2.cvtColor(self.colorsLAB[None,:,:],cv2.COLOR_RGB2Lab))
def sample_normal(self, col_mean, col_std):
"""
sample from a normal distribution centered around COL_MEAN
with standard deviation = COL_STD.
"""
col_sample = col_mean + col_std * np.random.randn()
return np.clip(col_sample, 0, 255).astype('uint8')
def sample_from_data(self,bg_mat):
"""
bg_mat : this is a nxmx3 RGB image.
returns a tuple : (RGB_foreground, RGB_background)
each of these is a 3-vector.
"""
bg_orig = bg_mat.copy()
bg_mat = cv2.cvtColor(bg_mat, cv2.COLOR_RGB2Lab)
bg_mat = np.reshape(bg_mat, (np.prod(bg_mat.shape[:2]),3))
bg_mean = np.mean(bg_mat,axis=0)
norms = np.linalg.norm(self.colorsLAB-bg_mean[None,:], axis=1)
# choose a random color amongst the top 3 closest matches:
#nn = np.random.choice(np.argsort(norms)[:3])
nn = np.argmin(norms)
## nearest neighbour color:
data_col = self.colorsRGB[np.mod(nn,self.ncol),:]
col1 = self.sample_normal(data_col[:3],data_col[3:6])
col2 = self.sample_normal(data_col[6:9],data_col[9:12])
if nn < self.ncol:
return (col2, col1)
else:
# need to swap to make the second color close to the input backgroun color
return (col1, col2)
if __name__ =='__main__':
fg_col,bg_col = sample_from_data(bgi)
# -*- coding: utf-8 -*-
#pygame replace Image
import os
import cv2
import glob
import math
import random
import numpy as np
import os.path as osp
from xml.dom.minidom import Document
import multiprocessing as mp
import logging
from PIL import Image,ImageDraw,ImageFont
import secrets
import pygame
from pygame.locals import *
from pygame import freetype
import get_color
resultImgsDir = '/home/result_imgs' # 生成的图片存放位置
resultXmlDir = '/home/result_xmls' # 生产的xml存放位置
bgiDir = '/home/background_images' # 添加背景图片
gTtf= '/home/ttfs' # 添加字体库
totalFile = '/home/zzc/data/synth_recepit_text/result_200.txt' # 所需要添加的文字，一行一句（或者一行一个单词）
FORMAT = '%(asctime)-15s [%(processName)s] %(message)s'
logging.basicConfig(format = FORMAT)
gBlockSize = 20 #每一个进程一次处理的句子
ttfSize = [28,30,35,40,45,50,55,60,65]
#====test
#charset = [line.strip().split('\t')[1] for line in open('text/chars_gb2312').readlines()[:-1]]
def _addSaltNoise(block,level = 10):
'''添加椒盐噪声 '''
ran = np.random.randint(0,level,block.shape)
salt = ran == 0
pepper = ran == level
block[salt]= 0
block[pepper] = 255
return block
def _addNoise(block,below=4,high =20):
''' 添加噪声'''
randValue = np.random.randn(*block.shape)*np.random.randint(below,high)
block = block+randValue
block[block<0] = 0.0
block[block>255] = 255.0
block = block.astype('uint8')
return block
def _feather(block, height):
''' 对图片进行羽化'''
# determine the gaussian-blur std:
if height <= 30 :
bsz = 0.25
ksz=1
elif 30 < height < 50:
bsz = max(0.30, 0.5 + 0.1*np.random.randn())
ksz = 3
else:
bsz = max(0.5, 1.5 + 0.5*np.random.randn())
ksz = 5#np.random.choice([1,3,5])#5
return cv2.GaussianBlur(block,(ksz,ksz),bsz)
def _seamlessClone(obj,dst,center):
''' 进行前背景合成'''
mask = 255 * np.ones(obj.shape, obj.dtype)
#print('obj,shape:',obj.shape,' dst.shape:',dst.shape,' center:',center)
try:
mixed_clone = cv2.seamlessClone(obj, dst, mask, center, cv2.MIXED_CLONE)
except Exception as e:
print('exception:',obj.shape,dst.shape,mask.shape,center)
raise e
return mixed_clone
def _rander(bgiGame,string,rowStart,font,get_color):
''' 进行渲染'''
isFailed = False
width, height = bgiGame.get_size()
'''sample the color '''
bgiNp = pygame.surfarray.array3d(bgiGame)
fg_col,bg_col = get_color.sample_from_data(bgiNp)
#fg_col = fg_col + np.random.randint(-3,3,[1,3])
fg_col = fg_col.squeeze()
'''change the property of font '''
font.oblique = secrets.choice([False,True])
font.rotation = secrets.choice(range(-5,5))
test = font.render(string)
txtwidth,txtheight = test[1].size
if width-txtwidth < 0: isFailed = True
colStart = secrets.randbelow(max(1, width-txtwidth))
if rowStart+txtheight > height or colStart+txtwidth>width or isFailed:
return bgiGame,rowStart,0,0,0
'''render the text '''
try:
font.render_to(bgiGame,(colStart,rowStart), string, fg_col)
except:
print('fg_col',fg_col)
'''surface 2 numpy '''
bgiNp = pygame.surfarray.array3d(bgiGame)
bgiNp = cv2.cvtColor(bgiNp.transpose([1,0,2]),cv2.COLOR_RGB2BGR)
'''add noise and blur '''
block = bgiNp[rowStart:rowStart+txtheight,colStart:colStart+txtwidth,:]
block = _addNoise(block,4,20)
if secrets.choice(range(4))==0: block = _addSaltNoise(block,np.random.randint(70,80))
block = _feather(block,txtheight)
block = _addNoise(block,2,20)
if secrets.choice(range(4))==0: block = _addSaltNoise(block,np.random.randint(70,80))
#=====
bgiNp[rowStart:rowStart+txtheight,colStart:colStart+txtwidth,:] = block
'''numpy 2 surface '''
bgiNp = cv2.cvtColor(bgiNp.transpose([1,0,2]),cv2.COLOR_BGR2RGB)
bgiGame = pygame.surfarray.make_surface(bgiNp)
return bgiGame,rowStart,colStart,txtwidth,txtheight
'''
bgi = _seamlessClone(block,bgi,center)
return bgi
'''
def _paste(bgiGame,ttf,size,rowStart,curText,cols,get_color):
#ttfont = ImageFont.truetype(ttf,size)
ttfont = freetype.Font(ttf,size)
curText = curText.strip()
'''random the digit '''
numberLength = 10
digits = ['0','1','2','3','4','5','6','7','8','9']
if secrets.randbelow(numberLength) == 0:
#curText = ''.join([str(random.randint(0,9)) for _ in range(shouldMaxNumTxt)])
curText = ''.join([secrets.choice(digits) for _ in range(numberLength)] )
string = curText
'''random the dot '''
if secrets.randbelow(numberLength-2) == 0:
dotInd = random.randint(1,numberLength-2)
string = curText[:dotInd]+'.'+curText[dotInd+1:]
else:
string = curText
'''如果maxNumText小于10，则跳过 '''
numText = len(string)
if numText != numberLength:
string = ''
return None,None,None,None
bgiGame,rowStart,colStart,txtwidth,txtheight = _rander(bgiGame,string,rowStart,ttfont,get_color)
return bgiGame,string,rowStart,colStart,txtwidth,txtheight
def _xml(doc,anno,string,xminT,yminT,xmaxT,ymaxT):
''' 生成对应的xml'''
if not string: return
body = doc.createElement('object')
anno.appendChild(body)
name = doc.createElement('name')
nameText = doc.createTextNode('text')
name.appendChild(nameText)
body.appendChild(name)
content = doc.createElement('textContent')
contentText = doc.createTextNode(string)
content.appendChild(contentText)
body.appendChild(content)
bndbox = doc.createElement('bndbox')
xmin = doc.createElement('xmin')
ymin = doc.createElement('ymin')
xmax = doc.createElement('xmax')
ymax = doc.createElement('ymax')
xminText = doc.createTextNode(str(xminT))
yminText = doc.createTextNode(str(yminT))
xmaxText = doc.createTextNode(str(xmaxT))
ymaxText = doc.createTextNode(str(ymaxT))
xmin.appendChild(xminText)
ymin.appendChild(yminText)
xmax.appendChild(xmaxText)
ymax.appendChild(ymaxText)
bndbox.appendChild(xmin)
bndbox.appendChild(ymin)
bndbox.appendChild(xmax)
bndbox.appendChild(ymax)
body.appendChild(bndbox)
def paste(imgname,bgi,text,ttf,get_color):
pygame.init()
bgiGame = pygame.image.load(bgi)
width,height = bgiGame.get_size()
depth = bgiGame.get_bitsize()//8
# 选择当前行的间距
curRow = 0
curRowInter = random.randint(3,7)
curRow += curRowInter
# 随机选择字体大小
curTtfSize = random.choice(ttfSize)
# 创建xml的文件头
doc = Document()
anno = doc.createElement('Annotations')
doc.appendChild(anno)
imgNameNode = doc.createElement('imgName')
imgNameNode.appendChild(doc.createTextNode(imgname))
anno.appendChild(imgNameNode)
sizeNode = doc.createElement('size')
widthNode = doc.createElement('width')
widthNode.appendChild(doc.createTextNode(str(width)))
sizeNode.appendChild(widthNode)
heightNode = doc.createElement('height')
heightNode.appendChild(doc.createTextNode(str(height)))
sizeNode.appendChild(heightNode)
depthNode = doc.createElement('depth')
depthNode.appendChild(doc.createTextNode(str(depth)))
sizeNode.appendChild(depthNode)
anno.appendChild(sizeNode)
# 循环的一行一行去将文字粘贴到对应的图片上
curCol = 0; numTextDone = 0
while curRow+curTtfSize <= width:
# cur col point
# cur row point
'''paste the text on bgiGame '''
if curRow+curTtfSize <= width:
# if curcols is bigger than 0.9*cols,then do not paste the line
curText = secrets.choice(text)
bgiGame,string,curRow,colStart,txtwidth,txtheight = _paste(bgiGame,ttf,curTtfSize,curRow,curText,width,get_color)
if not string: continue
numTextDone += 1
_xml(doc,anno,string,xminT = colStart,yminT = curRow,xmaxT = colStart+txtwidth,ymaxT = curRow+txtheight)
curRow += txtheight
curRow += curRowInter
# cur intervel
curRowInter = random.randint(3,6)
# cur ttf size
curTtfSize = random.choice(ttfSize)
bgi = pygame.surfarray.array3d(bgiGame).transpose([1,0,2])
bgi = cv2.cvtColor(bgi,cv2.COLOR_RGB2BGR)
return bgi, doc, numTextDone
def handle(indTexts):
ind, texts = indTexts
# 获取进程号
pid = os.getpid()
# 随机获取颜色
getcolor = get_color.ColorSample()
bgis = glob.glob( osp.join(bgiDir, '*.jpg') )
# 随机选择当前一张背景图
bgipath = random.choice(bgis)
# 随机获取字体
ttf = random.choice(ttfs)
# 调用paste函数进行操作
imgname = 'bgi{}_ind{}_pid{}_ttf{}.jpg'.format(osp.basename(bgipath),ind,pid,osp.basename(ttf))
bgiNp,doc,numTextDone = paste(imgname,bgipath,texts,ttf,getcolor)
imgnamep = 'bgi{}_ind{}_{}Of{}_ttf{}.jpg'.format(osp.basename(bgipath),ind,numTextDone,len(texts),osp.basename(ttf))
logging.warn(imgnamep)
# 将图片和xml写入到对应位置
cv2.imwrite(osp.join(resultImgsDir,imgname),bgiNp)
xmlFileName = osp.join(resultXmlDir,'{}.xml'.format(imgname[:-4]))
with open(xmlFileName, "w") as fxml:
fxml.write(str(doc.toprettyxml(indent = " ", newl = "\n", encoding = "utf-8"),encoding = 'utf-8'))
pygame.quit()
return
if __name__ == '__main__':
''' 1 - 先读取文字行，然后按照进程个数进行划分'''
total = [line.strip() for line in open(totalFile)]
numP = 30
totalSP = []
inter = math.ceil(len(total)/gBlockSize)
for i in range(inter):
totalSP.append(total[i::inter])
'''2 - 开启多进程进行处理 '''
print('begin',len(totalSP))
p = mp.Pool(numP)
p.map(handle, enumerate(totalSP))
结果如图：
bgi39.jpg_ind0_pid8387_ttf锐字工房云字库小标宋GBK.ttf.jpg
bgi39.jpg_ind0_pid8387_ttf锐字工房云字库小标宋GBK.ttf.xml
下面是采用PIL和opencv的seamlessClone粘贴方式，只是PIL这个包进行文字粘贴的时候，不支持文字旋转，且简单的文字粘贴，好像pygame的结果和seamlessClone效果差不多。
# -*- coding: utf-8 -*-
import os
import cv2
import glob
import math
import random
import numpy as np
import os.path as osp
from xml.dom.minidom import Document
import multiprocessing as mp
import logging
from PIL import Image,ImageDraw,ImageFont
import pygame
from pygame.locals import *
from pygame import freetype
import get_color
resultImgsDir = 'crnn_result_imgs1'
resultXmlDir = 'crnn_result_xmls1'
bgiDir = 'bgi'
gTtf= 'ttfs'
totalFile = 'texts.txt'
FORMAT = '%(asctime)-15s [%(processName)s] %(message)s'
logging.basicConfig(format = FORMAT)
gBlockSize = 20#num of each process's sentences
ttfSize = [28,30,35,40,45,50,55,60,65]
def _addSaltNoise(block,level = 10):
ran = np.random.randint(0,level,block.shape)
salt = ran == 0
pepper = ran == level
block[salt]= 0
block[pepper] = 255
return block
def _addNoise(block):
randValue = np.random.randn(*block.shape)*np.random.randint(2,20)
block = block+randValue
block[block<0] = 0.0
block[block>255] = 255.0
block = block.astype('uint8')
return block
def _feather(block, height):
# determine the gaussian-blur std:
if height <= 30 :
bsz = 0.25
ksz=1
elif 30 < height < 50:
bsz = max(0.30, 0.5 + 0.1*np.random.randn())
ksz = 3
else:
bsz = max(0.5, 1.5 + 0.5*np.random.randn())
ksz = 5#np.random.choice([1,3,5])#5
return cv2.GaussianBlur(block,(ksz,ksz),bsz)
def _seamlessClone(obj,dst,center):
mask = 255 * np.ones(obj.shape, obj.dtype)
#print('obj,shape:',obj.shape,' dst.shape:',dst.shape,' center:',center)
try:
mixed_clone = cv2.seamlessClone(obj, dst, mask, center, cv2.MIXED_CLONE)
except Exception as e:
print('exception:',obj.shape,dst.shape,mask.shape,center)
raise e
return mixed_clone
def _rander(rawbgi,string,bgr,point,font,get_color):
bgi = Image.fromarray(rawbgi)
draw = ImageDraw.Draw(bgi)
curCol,curRow = point
fg_col,bg_col = get_color.sample_from_data(rawbgi)
fg_col = fg_col + np.random.randint(-3,3,[1,3])
draw.text((curCol,curRow),string, tuple(fg_col.squeeze()), font=font)
width,height = font.getsize(string)
region = curCol,curRow,curCol+width,curRow+height
bgi = np.array(bgi)
block = bgi[curRow:curRow+height,curCol:curCol+width,:]
block = _addNoise(block)
block = _feather(block,height)
block = _addNoise(block)
block = _addSaltNoise(block,50)
#=====
# bgi[curRow:curRow+height,curCol:curCol+width,:] = block
# return bgi
#cv2.imwrite('/home/zzc/tmp111.jpg',block)
center = (curCol+curCol+width)//2,(curRow+curRow+height)//2
# width, height, channels = bgi.shape
# center = height//2,width//2
bgi = _seamlessClone(block,bgi,center)
return bgi
def _paste(bgi,ttf,size,curRow,curCol,curText,cols,get_color):
ttfont = ImageFont.truetype(ttf,size)
maxNumText = math.floor((cols-curCol)/size)
curText = curText.strip()
'''random the digit '''
shouldMaxNumTxt = 10
if random.randint(0,9)==9 and maxNumText >= 4:
curText = ''.join([str(random.randint(0,9)) for _ in range(shouldMaxNumTxt)])
string = curText
'''random the dot '''
if random.randint(0,7)==7:
dotInd = random.randint(1,shouldMaxNumTxt-2)
string = curText[:dotInd]+'.'+curText[dotInd+1:]
else:
startInd = random.randint(0,max(0,len(curText)-shouldMaxNumTxt-1))
string = curText[startInd:startInd+shouldMaxNumTxt].strip()
string= curText
'''如果maxNumText小于10，则跳过 '''
if maxNumText < 10 or len(curText)<10: string = ''
numText = len(string)
if numText == 10 :
bgr = [random.randint(100,254) for i in range(3)]
bgi = _rander(bgi,string,bgr,(curCol,curRow),ttfont,get_color)
else:
string = ''
#=====
'''get printed width height '''
width,height = ttfont.getsize(string)
return bgi,string,width,height
def _xml(doc,anno,string,xminT,yminT,xmaxT,ymaxT):
if not string: return
body = doc.createElement('object')
anno.appendChild(body)
name = doc.createElement('name')
nameText = doc.createTextNode('text')
name.appendChild(nameText)
body.appendChild(name)
content = doc.createElement('textContent')
contentText = doc.createTextNode(string)
content.appendChild(contentText)
body.appendChild(content)
bndbox = doc.createElement('bndbox')
xmin = doc.createElement('xmin')
ymin = doc.createElement('ymin')
xmax = doc.createElement('xmax')
ymax = doc.createElement('ymax')
xminText = doc.createTextNode(str(xminT))
yminText = doc.createTextNode(str(yminT))
xmaxText = doc.createTextNode(str(xmaxT))
ymaxText = doc.createTextNode(str(ymaxT))
xmin.appendChild(xminText)
ymin.appendChild(yminText)
xmax.appendChild(xmaxText)
ymax.appendChild(ymaxText)
bndbox.appendChild(xmin)
bndbox.appendChild(ymin)
bndbox.appendChild(xmax)
bndbox.appendChild(ymax)
body.appendChild(bndbox)
def paste(imgname,bgi,text,ttf,ttfRandom,get_color):
bgi = cv2.imread(bgi)
rows,cols,depth = bgi.shape
# bgi = Image.fromarray(bgi)
# draw = ImageDraw.Draw(bgi)
curRow = 0
curRowInter = random.randint(3,7)
curRow += curRowInter
curTtfSize = random.randint(0,len(ttfRandom)-1)
#create the xml head
doc = Document()
anno = doc.createElement('Annotations')
doc.appendChild(anno)
imgNameNode = doc.createElement('imgName')
imgNameNode.appendChild(doc.createTextNode(imgname))
anno.appendChild(imgNameNode)
height,width,depth = rows,cols,depth
sizeNode = doc.createElement('size')
widthNode = doc.createElement('width')
widthNode.appendChild(doc.createTextNode(str(width)))
sizeNode.appendChild(widthNode)
heightNode = doc.createElement('height')
heightNode.appendChild(doc.createTextNode(str(height)))
sizeNode.appendChild(heightNode)
depthNode = doc.createElement('depth')
depthNode.appendChild(doc.createTextNode(str(depth)))
sizeNode.appendChild(depthNode)
anno.appendChild(sizeNode)
while curRow + ttfRandom[curTtfSize] <=rows:
#cur col point
curCol = random.randint(0,cols-1)
#cur row point
'''paste the text on bgi '''
if curCol < cols*0.9 and curRow+ttfRandom[curTtfSize] <= rows:
#if curcols is bigger than 0.9*cols,then do not paste the line
curText = text[random.randint(0,len(text)-1)]
bgi,string,width,height = _paste(bgi,ttf,ttfRandom[curTtfSize],curRow,curCol,curText,cols,get_color)
if not string: continue
_xml(doc,anno,string,xminT = curCol,yminT = curRow,xmaxT = curCol+width,ymaxT = curRow+height)
curRow += curRowInter
curRow += ttfRandom[curTtfSize]
#cur intervel
curRowInter = random.randint(3,7)
#cur ttf size
curTtfSize = random.randint(0,len(ttfRandom)-1)
return np.array(bgi), doc
def handle(text):
ind, text = text
#pid
pid = os.getpid()
#background image
getcolor = get_color.ColorSample()
bgis = glob.glob( osp.join(bgiDir,'*.jpg') )
#select one background image
curBgi = random.randint(0,len(bgis)-1)
bgi = bgis[curBgi]
#ttf
ttfs = glob.glob(osp.join(gTtf,'*.ttf'))
curTtf = random.randint(0,len(ttfs)-1)
ttf = ttfs[curTtf]
#ttf size random
ttfRandom = [1]+[ random.randint(0,1) for i in range(len(ttfSize)-1)]
ttfRandom = [ran*size for ran,size in zip(ttfRandom, ttfSize)]
ttfRandom = [i for i in ttfRandom if i != 0]
imgname = '{}_{}_{}.jpg'.format(ind,pid,curTtf)
bgi,doc = paste(imgname,bgi,text,ttf,ttfRandom,getcolor)
cv2.imwrite(osp.join(resultImgsDir,imgname),bgi)
xmlFileName = osp.join(resultXmlDir,'{}.xml'.format(imgname[:-4]))
with open(xmlFileName, "w") as fxml:
fxml.write(str(doc.toprettyxml(indent = " ", newl = "\n", encoding = "utf-8"),encoding = 'utf-8'))
logging.warn('{}'.format(ind))
return
if __name__ == '__main__':
total = [line.strip() for line in open(totalFile)]
numP = 30
totalSP = []
inter = math.ceil(len(total)/gBlockSize)
for i in range(inter):
totalSP.append(total[i::inter])
print('begin')
p = mp.Pool(numP)
p.map(handle, enumerate(totalSP[:1000]))
