GapClassification-CharClass.ipynb

Kernel: Python 3

Character Segmentation using Char Classifier

This is experimental solution, UNFINISHED

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import cv2

# Import Widgets
import ipywidgets as widgets
from IPython.display import display, clear_output
from operator import itemgetter
import unidecode

# Import costume functions, corresponding to notebooks
from ocr import page, words
from ocr.normalization import imageNorm, letterNorm
from ocr.tfhelpers import Graph
from ocr.datahelpers import idx2char
# Helper functions - ploting and resizing
from ocr.helpers import implt, resize

# plt.rcParams['figure.figsize'] = (15, 9)

print("OpenCV: " + cv2.__version__)
print("Numpy: " + np.__version__)
print("TensorFlow: " + tf.__version__)

/home/breta/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters

OpenCV: 3.1.0
Numpy: 1.14.1
TensorFlow: 1.6.0

plt.rcParams['figure.figsize'] = (15, 9)

Gloval Variables

IMG = "wordTest-en"
LANG = 'en'
CLASS = 53

Load Image and Separate Words

image = cv2.cvtColor(cv2.imread("data/pagedet/%s.jpg" % IMG), cv2.COLOR_BGR2RGB)
implt(image)

# Crop image and get bounding boxes of words
crop = page.detection(image)
implt(crop)
bBoxes = words.detection(crop)

Load Trained Model

gapClass = Graph('models/gap-clas/CNN-CG')
charClass = Graph('models/char-clas/' + LANG + '/CharClassifier', operation='y_conv')
print("Successfully loaded.")

INFO:tensorflow:Restoring parameters from models/gap-clas/CNN-CG
INFO:tensorflow:Restoring parameters from models/char-clas/en/CharClassifier
Successfully loaded.

WORDS = []
with open('data/' + LANG + '_50k.txt') as f:
    for line in f:
        if LANG == 'en':
            WORDS += [unidecode.unidecode(line.split(" ")[0])]
        else:
            WORDS += [line.split(" ")[0]]

print(WORDS[:10])

['you', 'i', 'the', 'to', 'a', 'it', 'and', 'that', 'of', 'is']

Applying Model

class Cycler:
    """ Cycle through boxes, separate words """
    height = 60
    step = 2
    
    def __init__(self, image, boxes, idx):
        self.boxes = boxes       # Array of bounding boxes
        self.image = image       # Whole image
        self.index = idx         # Index of current bounding box

        self.nextImg()
    
    def separateWord(self, img):
        """ Separating word into letters """
        implt(img, 'gray')
        self.img = img
        self.run = True
        
        self.recWordFind('', 0)
        
#         print(gaps)
#         img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
#         for gap in gaps:
#             cv2.line(img, (gap,0),(gap,60),(0,255,0),1)
        
#         implt(img)
        
    def recWordFind(self, word, x):
        if self.run:
            if x + 10 < self.img.shape[1]:
                gaps = self.findMax(x)
                for gap in gaps:
                    if gap[1][0] > 0.7 and (gap[0].islower() or x == 0):
                        self.recWordFind(word+gap[0], x+gap[1][1])
            else:
                if word.lower() in WORDS:
#                     print('RESULT:')
#                     self.run = False
                    print(word)

        
    def findMax(self, x):
        """ Find max letter from given position x """        
        offset = 6    # Test different values
        
        idx = 0
        idx += offset
        
        d = {}
        for i in range(CLASS):
            d[idx2char(i)] = [0, 0]
            
        while (x + idx <= self.img.shape[1] and idx <= 120):
            crop = self.img[:, x:x + idx]
            char, dim = letterNorm(crop, is_thresh=True, dim=True)
            if dim[0] > 4 and dim[1] > 4:
                values = charClass.run([char.flatten()])
                
                sm = self.softmax(values)[0]
                l = idx2char(np.argmax(sm))
                d[l] = d[l] if d[l][0] > max(sm) else [max(sm), idx]
                
            idx += 2

        top3 = sorted(d.items(), key=itemgetter(1), reverse=True)[:5]
        #print(top3)
        #print()
        return top3
    
            
    def softmax(self, x):
        """ Compute softmax values for each sets of scores in x """
        return np.exp(x) / np.sum(np.exp(x), axis=1)
            

    def nextImg(self, btn=None):
        """ Getting next image from the array """
        clear_output()
        
        if self.index < len(self.boxes):
            b = self.boxes[self.index]
            x1, y1, x2, y2 = b
            
            # Cuting out the word image and resizing to standard height
            img = resize(self.image[y1:y2, x1:x2], self.height, True)                      
            implt(img, t='Original')  

            self.separateWord(
                imageNorm(
                    img,
                    self.height,
                    border=False,
                    tilt=True,
                    hystNorm=True))

            # Printing index for recovery
            print("Index: " + str(self.index))
            
            # Create button for cycling through images
            bNexi = widgets.Button(description="Next Image")
            bNexi.on_click(self.nextImg)
            display(bNexi)
            
            self.index += 1
            return 0
        else:
            print("END")
            return -1

# Class cycling through text positions
# Green - Characters
# Red   - Gaps
LAST_INDEX = 10
cycler = Cycler(crop, bBoxes, LAST_INDEX)

address
aum
muss