tesseract::Dict Class Reference

#include <dict.h>

Public Member Functions
	Dict (Image *image_ptr)
	~Dict ()
const Image *	getImage () const
Image *	getImage ()
const UNICHARSET &	getUnicharset () const
UNICHARSET &	getUnicharset ()
const UnicharAmbigs &	getUnicharAmbigs ()
bool	compound_marker (UNICHAR_ID unichar_id)
bool	hyphenated () const
	Returns true if we've recorded the beginning of a hyphenated word. More...
int	hyphen_base_size () const
	Size of the base word (the part on the line before) of a hyphenated word. More...
void	copy_hyphen_info (WERD_CHOICE *word) const
void	remove_hyphen_head (WERD_CHOICE *word) const
bool	has_hyphen_end (UNICHAR_ID unichar_id, bool first_pos) const
	Check whether the word has a hyphen at the end. More...
bool	has_hyphen_end (const WERD_CHOICE &word) const
	Same as above, but check the unichar at the end of the word. More...
void	reset_hyphen_vars (bool last_word_on_line)
void	set_hyphen_word (const WERD_CHOICE &word, const DawgInfoVector &active_dawgs, const DawgInfoVector &constraints)
void	update_best_choice (const WERD_CHOICE &word, WERD_CHOICE *best_choice)
void	init_active_dawgs (int sought_word_length, DawgInfoVector *active_dawgs, bool ambigs_mode) const
void	init_constraints (DawgInfoVector *constraints) const
bool	ambigs_mode (float rating_limit)
	Returns true if we are operating in ambigs mode. More...
WERD_CHOICE *	dawg_permute_and_select (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit)
WERD_CHOICE *	get_top_choice_word (const BLOB_CHOICE_LIST_VECTOR &char_choices)
WERD_CHOICE *	permute_top_choice (const BLOB_CHOICE_LIST_VECTOR &char_choices, float *rating_limit, WERD_CHOICE *raw_choice, BOOL8 *any_alpha)
WERD_CHOICE *	permute_all (const BLOB_CHOICE_LIST_VECTOR &char_choices, const WERD_CHOICE *best_choice, WERD_CHOICE *raw_choice)
void	end_permute ()
void	permute_subword (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit, int start, int end, WERD_CHOICE *current_word)
bool	permute_characters (const BLOB_CHOICE_LIST_VECTOR &char_choices, WERD_CHOICE *best_choice, WERD_CHOICE *raw_choice)
WERD_CHOICE *	permute_compound_words (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit)
WERD_CHOICE *	permute_fixed_length_words (const BLOB_CHOICE_LIST_VECTOR &char_choices, PermuterState *permuter_state)
void	incorporate_segcost (WERD_CHOICE *word)
	Incoporate segmentation cost into word rating. More...
WERD_CHOICE *	permute_script_words (const BLOB_CHOICE_LIST_VECTOR &char_choices, PermuterState *permuter_state)
WERD_CHOICE *	permute_chartype_words (const BLOB_CHOICE_LIST_VECTOR &char_choices, PermuterState *permuter_state)
	checks for consistency in character property (eg. alpah, digit, punct) More...
char	top_word_chartype (const BLOB_CHOICE_LIST_VECTOR &char_choices, char *pos_chartypes)
bool	NoDangerousAmbig (WERD_CHOICE *BestChoice, DANGERR *fixpt, bool fix_replaceable, BLOB_CHOICE_LIST_VECTOR *Choices, bool *modified_blobs)
double	StopperAmbigThreshold (double f1, double f2)
int	FreeBadChoice (void *item1, void *item2)
void	ReplaceAmbig (int wrong_ngram_begin_index, int wrong_ngram_size, UNICHAR_ID correct_ngram_id, WERD_CHOICE *werd_choice, BLOB_CHOICE_LIST_VECTOR *blob_choices, bool *modified_blobs)
void	DisableChoiceAccum ()
void	EnableChoiceAccum ()
bool	ChoiceAccumEnabled ()
int	LengthOfShortestAlphaRun (const WERD_CHOICE &WordChoice)
	Returns the length of the shortest alpha run in WordChoice. More...
VIABLE_CHOICE	NewViableChoice (const WERD_CHOICE &WordChoice, FLOAT32 AdjustFactor, const float Certainties[])
void	PrintViableChoice (FILE *File, const char *Label, VIABLE_CHOICE Choice)
	Dumps a text representation of the specified Choice to File. More...
bool	StringSameAs (const WERD_CHOICE &WordChoice, VIABLE_CHOICE ViableChoice)
bool	StringSameAs (const char *String, const char *String_lengths, VIABLE_CHOICE ViableChoice)
	Compares String to ViableChoice and returns true if they are the same. More...
int	UniformCertainties (const BLOB_CHOICE_LIST_VECTOR &Choices, const WERD_CHOICE &BestChoice)
bool	AcceptableChoice (BLOB_CHOICE_LIST_VECTOR *Choices, WERD_CHOICE *BestChoice, DANGERR *fixpt, ACCEPTABLE_CHOICE_CALLER caller, bool *modified_blobs)
	Returns true if the given best_choice is good enough to stop. More...
bool	AcceptableResult (const WERD_CHOICE &BestChoice)
int	ChoiceSameAs (const WERD_CHOICE &WordChoice, VIABLE_CHOICE ViableChoice)
void	LogNewChoice (FLOAT32 AdjustFactor, const float Certainties[], bool raw_choice, WERD_CHOICE *WordChoice, const BLOB_CHOICE_LIST_VECTOR &blob_choices)
void	EndDangerousAmbigs ()
bool	CurrentBestChoiceIs (const WERD_CHOICE &WordChoice)
	Returns true if WordChoice is the same as the current best choice. More...
FLOAT32	CurrentBestChoiceAdjustFactor ()
	Returns the adjustment factor for the best choice for the current word. More...
bool	CurrentWordAmbig ()
	Returns true if there are multiple good choices for the current word. More...
void	DebugWordChoices ()
	Prints the current choices for this word to stdout. More...
void	PrintAmbigAlternatives (FILE *file, const char *label, int label_num_unichars)
	Print all the choices in raw_choices_ list for non 1-1 ambiguities. More...
void	FillViableChoice (const WERD_CHOICE &WordChoice, FLOAT32 AdjustFactor, const float Certainties[], VIABLE_CHOICE ViableChoice)
bool	AlternativeChoicesWorseThan (FLOAT32 Threshold)
void	FilterWordChoices ()
void	FindClassifierErrors (FLOAT32 MinRating, FLOAT32 MaxRating, FLOAT32 RatingMargin, FLOAT32 Thresholds[])
void	InitChoiceAccum ()
void	ClearBestChoiceAccum ()
	Clears best_choices_ list accumulated by the stopper. More...
void	LogNewSegmentation (PIECES_STATE BlobWidth)
void	LogNewSplit (int Blob)
void	AddNewChunk (VIABLE_CHOICE Choice, int Blob)
void	SettupStopperPass1 ()
	Sets up stopper variables in preparation for the first pass. More...
void	SettupStopperPass2 ()
	Sets up stopper variables in preparation for the second pass. More...
int	case_ok (const WERD_CHOICE &word, const UNICHARSET &unicharset)
	Check a string to see if it matches a set of lexical rules. More...
bool	absolute_garbage (const WERD_CHOICE &word, const UNICHARSET &unicharset)
void	Load ()
void	End ()
void	ResetDocumentDictionary ()
void	LoadEquivalenceList (const char *unichar_strings[])
UNICHAR_ID	NormalizeUnicharIdForMatch (UNICHAR_ID unichar_id) const
int	def_letter_is_okay (void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const
int	LetterIsOkay (void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const
	Calls letter_is_okay_ member function. More...
double	ProbabilityInContext (const char *context, int context_bytes, const char *character, int character_bytes)
	Calls probability_in_context_ member function. More...
double	def_probability_in_context (const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)
	Default (no-op) implementation of probability in context function. More...
double	ngram_probability_in_context (const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)
const int	NumDawgs () const
	Return the number of dawgs in the dawgs_ vector. More...
const Dawg *	GetDawg (int index) const
	Return i-th dawg pointer recorded in the dawgs_ vector. More...
const Dawg *	GetPuncDawg () const
	Return the points to the punctuation dawg. More...
const Dawg *	GetUnambigDawg () const
	Return the points to the unambiguous words dawg. More...
const Dawg *	GetFixedLengthDawg (int word_length) const
	Return the pointer to the Dawg that contains words of length word_length. More...
const int	GetMaxFixedLengthDawgIndex () const
bool	ConstraintsOk (const DawgInfoVector &constraints, int word_end, DawgType current_dawg_type) const
void	ProcessPatternEdges (const Dawg *dawg, const DawgInfo &info, UNICHAR_ID unichar_id, bool word_end, DawgArgs *dawg_args, PermuterType *current_permuter) const
int	valid_word (const WERD_CHOICE &word, bool numbers_ok) const
int	valid_word (const WERD_CHOICE &word) const
int	valid_word_or_number (const WERD_CHOICE &word) const
int	valid_word (const char *string) const
	This function is used by api/tesseract_cube_combiner.cpp. More...
bool	valid_bigram (const WERD_CHOICE &word1, const WERD_CHOICE &word2) const
bool	valid_punctuation (const WERD_CHOICE &word)
int	good_choice (const WERD_CHOICE &choice)
	Returns true if a good answer is found for the unknown blob rating. More...
void	add_document_word (const WERD_CHOICE &best_choice)
	Adds a word found on this document to the document specific dictionary. More...
int	get_top_word_script (const BLOB_CHOICE_LIST_VECTOR &char_choices, const UNICHARSET &unicharset)
void	adjust_word (WERD_CHOICE *word, float *certainty_array, const BLOB_CHOICE_LIST_VECTOR *char_choices, bool nonword, float additional_adjust, bool debug)
	Adjusts the rating of the given word. More...
void	adjust_word (WERD_CHOICE *word, float *certainty_array, const BLOB_CHOICE_LIST_VECTOR *char_choices, bool debug)
void	adjust_non_word (WERD_CHOICE *word, float *certainty_array, const BLOB_CHOICE_LIST_VECTOR *char_choices, bool debug)
void	SetWordsegRatingAdjustFactor (float f)
	Set wordseg_rating_adjust_factor_ to the given value. More...
const LIST &	getBestChoices ()
go_deeper_dawg_fxn
If the choice being composed so far could be a dictionary word keep exploring choices. There are two modes for deciding whether to go deeper: regular dawg permuter mode and the special ambigs mode. If *limit is <= 0.0 the function switches to the ambigs mode (this is the case when dawg_permute_and_select() function is called from NoDangerousAmbigs()) and only searches for the first choice that has a rating better than *limit (in this case ratings are fake, since the real ratings can not be < 0). Modification of the hyphen state is turned off in the ambigs mode. When in the regular dawg permuter mode, the function explores all the possible words and chooses the one with the best rating. The letters with ratings that are far worse than the ones seen so far are pruned out.
WERD_CHOICE *	dawg_permute_and_select (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit, int sought_word_length, int end_char_choice_index)
void	go_deeper_dawg_fxn (const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, bool word_ending, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *void_more_args)
choose_il1
Choose between the candidate il1 chars. Parameters first_char first choice second_char second choice third_char third choice prev_char prev in word next_char next in word next_next_char after next next in word
const char *	choose_il1 (const char *first_char, const char *second_char, const char *third_char, const char *prev_char, const char *next_char, const char *next_next_char)
fragment_state
Given the current char choice and information about previously seen fragments, determines whether adjacent character fragments are present and whether they can be concatenated. The given prev_char_frag_info contains: fragment: if not NULL contains information about immediately preceeding fragmented character choice num_fragments: number of fragments that have been used so far to construct a character certainty: certainty of the current choice or minimum certainty of all fragments concatenated so far rating: rating of the current choice or sum of fragment ratings concatenated so far The output char_frag_info is filled in as follows: character: is set to be NULL if the choice is a non-matching or non-ending fragment piece; is set to unichar of the given choice if it represents a regular character or a matching ending fragment fragment,num_fragments,certainty,rating are set as described above Returns false if a non-matching fragment is discovered, true otherwise.
WERD_CHOICE *	top_fragments_permute_and_select (const BLOB_CHOICE_LIST_VECTOR &char_choices, float rating_limit)
void	go_deeper_top_fragments_fxn (const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, bool word_ending, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *more_args)
bool	fragment_state_okay (UNICHAR_ID curr_unichar_id, float curr_rating, float curr_certainty, const CHAR_FRAGMENT_INFO *prev_char_frag_info, const char *debug, int word_ending, CHAR_FRAGMENT_INFO *char_frag_info)
	Semi-generic functions used by multiple permuters. More...
void	permute_choices (const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *more_args)
void	append_choices (const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, const BLOB_CHOICE &blob_choice, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *more_args)

Static Public Member Functions
static NODE_REF	GetStartingNode (const Dawg *dawg, EDGE_REF edge_ref)
	Returns the appropriate next node given the EDGE_REF. More...
static void	ReadFixedLengthDawgs (DawgType type, const STRING &lang, PermuterType perm, int debug_level, FILE *file, DawgVector *dawg_vec, int *max_wdlen)
static void	WriteFixedLengthDawgs (const GenericVector< SquishedDawg * > &dawg_vec, int num_dawgs, int debug_level, FILE *output_file)
static bool	valid_word_permuter (uinT8 perm, bool numbers_ok)
	Check all the DAWGs to see if this word is in any of them. More...

Public Attributes
void(Dict::*	go_deeper_fxn_ )(const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, bool word_ending, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *void_more_args)
	Pointer to go_deeper function that will be modified by various permuters. More...
int(Dict::*	letter_is_okay_ )(void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const
double(Dict::*	probability_in_context_ )(const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)
	Probability in context function used by the ngram permuter. More...
char *	user_words_suffix = ""
char *	user_patterns_suffix = ""
bool	load_system_dawg = true
bool	load_freq_dawg = true
bool	load_unambig_dawg = true
bool	load_punc_dawg = true
bool	load_number_dawg = true
bool	load_fixed_length_dawgs = true
bool	load_bigram_dawg = false
double	segment_penalty_dict_frequent_word = 1.0
double	segment_penalty_dict_case_ok = 1.1
double	segment_penalty_dict_case_bad = 1.3125
double	segment_penalty_ngram_best_choice = 1.24
double	segment_penalty_dict_nonword = 1.25
double	segment_penalty_garbage = 1.50
char *	output_ambig_words_file = ""
int	dawg_debug_level = 0
int	hyphen_debug_level = 0
int	max_viterbi_list_size = 10
bool	use_only_first_uft8_step = false
double	certainty_scale = 20.0
double	stopper_nondict_certainty_base = -2.50
double	stopper_phase2_certainty_rejection_offset = 1.0
int	stopper_smallword_size = 2
double	stopper_certainty_per_char = -0.50
double	stopper_allowable_character_badness = 3.0
int	stopper_debug_level = 0
bool	stopper_no_acceptable_choices = false
double	stopper_ambiguity_threshold_gain = 8.0
double	stopper_ambiguity_threshold_offset = 1.5
bool	save_raw_choices = false
int	tessedit_truncate_wordchoice_log = 10
char *	word_to_debug = ""
char *	word_to_debug_lengths = ""
int	fragments_debug = 0
int	segment_debug = 0
bool	permute_debug = 0
double	bestrate_pruning_factor = 2.0
bool	permute_script_word = 0
bool	segment_segcost_rating = 0
bool	segment_nonalphabetic_script = false
double	segment_reward_script = 0.95
bool	permute_fixed_length_dawg = 0
bool	permute_chartype_word = 0
double	segment_reward_chartype = 0.97
double	segment_reward_ngram_best_choice = 0.99
bool	save_doc_words = 0
bool	doc_dict_enable = 1
double	doc_dict_pending_threshold = 0.0
double	doc_dict_certainty_threshold = -2.25
bool	ngram_permuter_activated = false
int	max_permuter_attempts = 10000
bool	permute_only_top = false

Detailed Description

Definition at line 90 of file dict.h.

Constructor & Destructor Documentation

tesseract::Dict::Dict

(

Image *

image_ptr

)

Definition at line 33 of file dict.cpp.

    : letter_is_okay_(&tesseract::Dict::def_letter_is_okay),
      probability_in_context_(&tesseract::Dict::def_probability_in_context),
      image_ptr_(image_ptr),
      STRING_INIT_MEMBER(user_words_suffix, "",
                         "A list of user-provided words.",
                         getImage()->getCCUtil()->params()),
      STRING_INIT_MEMBER(user_patterns_suffix, "",
                         "A list of user-provided patterns.",
                         getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_system_dawg, true, "Load system word dawg.",
                       getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_freq_dawg, true, "Load frequent word dawg.",
                       getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_unambig_dawg, true, "Load unambiguous word dawg.",
                       getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_punc_dawg, true, "Load dawg with punctuation"
                       " patterns.", getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_number_dawg, true, "Load dawg with number"
                       " patterns.", getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_fixed_length_dawgs, true, "Load fixed length dawgs"
                       " (e.g. for non-space delimited languages)",
                       getImage()->getCCUtil()->params()),
      BOOL_INIT_MEMBER(load_bigram_dawg, false, "Load dawg with special word "
                       "bigrams.", getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_dict_frequent_word, 1.0,
                    "Score multiplier for word matches which have good case and"
                    "are frequent in the given language (lower is better).",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_dict_case_ok, 1.1,
                    "Score multiplier for word matches that have good case "
                    "(lower is better).", getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_dict_case_bad, 1.3125,
                    "Default score multiplier for word matches, which may have "
                    "case issues (lower is better).",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_ngram_best_choice, 1.24,
                   "Multipler to for the best choice from the ngram model.",
                   getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_dict_nonword, 1.25,
                    "Score multiplier for glyph fragment segmentations which "
                    "do not match a dictionary word (lower is better).",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(segment_penalty_garbage, 1.50,
                    "Score multiplier for poorly cased strings that are not in"
                    " the dictionary and generally look like garbage (lower is"
                    " better).", getImage()->getCCUtil()->params()),
      STRING_MEMBER(output_ambig_words_file, "",
                    "Output file for ambiguities found in the dictionary",
                    getImage()->getCCUtil()->params()),
      INT_MEMBER(dawg_debug_level, 0, "Set to 1 for general debug info"
                 ", to 2 for more details, to 3 to see all the debug messages",
                 getImage()->getCCUtil()->params()),
      INT_MEMBER(hyphen_debug_level, 0, "Debug level for hyphenated words.",
                 getImage()->getCCUtil()->params()),
      INT_MEMBER(max_viterbi_list_size, 10, "Maximum size of viterbi list.",
                 getImage()->getCCUtil()->params()),
      BOOL_MEMBER(use_only_first_uft8_step, false,
                  "Use only the first UTF8 step of the given string"
                  " when computing log probabilities.",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(certainty_scale, 20.0, "Certainty scaling factor",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_nondict_certainty_base, -2.50,
                    "Certainty threshold for non-dict words",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_phase2_certainty_rejection_offset, 1.0,
                    "Reject certainty offset",
                    getImage()->getCCUtil()->params()),
      INT_MEMBER(stopper_smallword_size, 2,
                 "Size of dict word to be treated as non-dict word",
                 getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_certainty_per_char, -0.50, "Certainty to add"
                    " for each dict char above small word size.",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_allowable_character_badness, 3.0,
                    "Max certaintly variation allowed in a word (in sigma)",
                    getImage()->getCCUtil()->params()),
      INT_MEMBER(stopper_debug_level, 0, "Stopper debug level",
                 getImage()->getCCUtil()->params()),
      BOOL_MEMBER(stopper_no_acceptable_choices, false,
                  "Make AcceptableChoice() always return false. Useful"
                  " when there is a need to explore all segmentations",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_ambiguity_threshold_gain, 8.0,
                    "Gain factor for ambiguity threshold.",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(stopper_ambiguity_threshold_offset, 1.5,
                    "Certainty offset for ambiguity threshold.",
                    getImage()->getCCUtil()->params()),
      BOOL_MEMBER(save_raw_choices, false, "Save all explored raw choices",
                  getImage()->getCCUtil()->params()),
      INT_MEMBER(tessedit_truncate_wordchoice_log, 10,
                 "Max words to keep in list",
                 getImage()->getCCUtil()->params()),
      STRING_MEMBER(word_to_debug, "", "Word for which stopper debug"
                    " information should be printed to stdout",
                    getImage()->getCCUtil()->params()),
      STRING_MEMBER(word_to_debug_lengths, "",
                    "Lengths of unichars in word_to_debug",
                    getImage()->getCCUtil()->params()),
      INT_MEMBER(fragments_debug, 0, "Debug character fragments",
                 getImage()->getCCUtil()->params()),
      INT_MEMBER(segment_debug, 0, "Debug the whole segmentation process",
                 getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_debug, 0, "Debug char permutation process",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(bestrate_pruning_factor, 2.0, "Multiplying factor of"
                    " current best rate to prune other hypotheses",
                    getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_script_word, 0,
                  "Turn on word script consistency permuter",
                  getImage()->getCCUtil()->params()),
      BOOL_MEMBER(segment_segcost_rating, 0,
                  "incorporate segmentation cost in word rating?",
                  getImage()->getCCUtil()->params()),
      BOOL_MEMBER(segment_nonalphabetic_script, false,
                 "Don't use any alphabetic-specific tricks."
                 "Set to true in the traineddata config file for"
                 " scripts that are cursive or inherently fixed-pitch",
                 getImage()->getCCUtil()->params()),
      double_MEMBER(segment_reward_script, 0.95,
                    "Score multipler for script consistency within a word. "
                    "Being a 'reward' factor, it should be <= 1. "
                    "Smaller value implies bigger reward.",
                    getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_fixed_length_dawg, 0,
                  "Turn on fixed-length phrasebook search permuter",
                  getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_chartype_word, 0,
                  "Turn on character type (property) consistency permuter",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(segment_reward_chartype, 0.97,
                    "Score multipler for char type consistency within a word. ",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(segment_reward_ngram_best_choice, 0.99,
                    "Score multipler for ngram permuter's best choice"
                    " (only used in the Han script path).",
                    getImage()->getCCUtil()->params()),
      BOOL_MEMBER(save_doc_words, 0, "Save Document Words",
                  getImage()->getCCUtil()->params()),
      BOOL_MEMBER(doc_dict_enable, 1, "Enable Document Dictionary ",
                  getImage()->getCCUtil()->params()),
      double_MEMBER(doc_dict_pending_threshold, 0.0,
                    "Worst certainty for using pending dictionary",
                    getImage()->getCCUtil()->params()),
      double_MEMBER(doc_dict_certainty_threshold, -2.25,
                    "Worst certainty for words that can be inserted into the"
                    "document dictionary", getImage()->getCCUtil()->params()),
      BOOL_MEMBER(ngram_permuter_activated, false,
                  "Activate character-level n-gram-based permuter",
                  getImage()->getCCUtil()->params()),
      INT_MEMBER(max_permuter_attempts, 10000, "Maximum number of different"
                 " character choices to consider during permutation."
                 " This limit is especially useful when user patterns"
                 " are specified, since overly generic patterns can result in"
                 " dawg search exploring an overly large number of options.",
                 getImage()->getCCUtil()->params()),
      BOOL_MEMBER(permute_only_top, false, "Run only the top choice permuter",
                  getImage()->getCCUtil()->params()) {
  dang_ambigs_table_ = NULL;
  replace_ambigs_table_ = NULL;
  keep_word_choices_ = false;
  reject_offset_ = 0.0;
  best_raw_choice_ = NULL;
  best_choices_ = NIL_LIST;
  raw_choices_ = NIL_LIST;
  go_deeper_fxn_ = NULL;
  hyphen_word_ = NULL;
  last_word_on_line_ = false;
  hyphen_unichar_id_ = INVALID_UNICHAR_ID;
  document_words_ = NULL;
  pending_words_ = NULL;
  bigram_dawg_ = NULL;
  freq_dawg_ = NULL;
  punc_dawg_ = NULL;
  max_fixed_length_dawgs_wdlen_ = -1;
  wordseg_rating_adjust_factor_ = -1.0f;
  output_ambig_words_file_ = NULL;
}

tesseract::Dict::~Dict

(

)

Definition at line 214 of file dict.cpp.

            {
  if (hyphen_word_ != NULL) delete hyphen_word_;
  if (output_ambig_words_file_ != NULL) fclose(output_ambig_words_file_);
}

Member Function Documentation

bool tesseract::Dict::absolute_garbage	(	const WERD_CHOICE &	word,
		const UNICHARSET &	unicharset
	)

Returns true if the word looks like an absolute garbage (e.g. image mistakenly recognized as text).

Definition at line 78 of file context.cpp.

                                                          {
  if (word.length() < kMinAbsoluteGarbageWordLength) return false;
  int num_alphanum = 0;
  for (int x = 0; x < word.length(); ++x) {
    num_alphanum += (unicharset.get_isalpha(word.unichar_id(x)) ||
                     unicharset.get_isdigit(word.unichar_id(x)));
  }
  return (static_cast<float>(num_alphanum) /
          static_cast<float>(word.length()) < kMinAbsoluteGarbageAlphanumFrac);
}

bool tesseract::Dict::AcceptableChoice	(	BLOB_CHOICE_LIST_VECTOR *	Choices,
		WERD_CHOICE *	BestChoice,
		DANGERR *	fixpt,
		ACCEPTABLE_CHOICE_CALLER	caller,
		bool *	modified_blobs
	)

Returns true if the given best_choice is good enough to stop.

Definition at line 191 of file stopper.cpp.

                                                  {
  float CertaintyThreshold = stopper_nondict_certainty_base;
  int WordSize;
  if (modified_blobs != NULL) *modified_blobs = false;

  if (stopper_no_acceptable_choices) return false;

  if (fixpt != NULL) fixpt->clear();
  if (BestChoice->length() == 0)
    return false;
  if (caller == CHOPPER_CALLER && BestChoice->fragment_mark()) {
    if (stopper_debug_level >= 1) {
      cprintf("AcceptableChoice(): a choice with fragments beats BestChoice");
    }
    return false;
  }

  bool no_dang_ambigs = (GetMaxFixedLengthDawgIndex() >= 0 ||
                         NoDangerousAmbig(BestChoice, fixpt, true,
                                          Choices, modified_blobs));
  bool is_valid_word = valid_word_permuter(BestChoice->permuter(), false);
  bool is_case_ok = case_ok(*BestChoice, getUnicharset());

  if (stopper_debug_level >= 1)
    tprintf("\nStopper:  %s (word=%c, case=%c)\n",
            BestChoice->debug_string().string(),
            (is_valid_word ? 'y' : 'n'),
            (is_case_ok ? 'y' : 'n'));

  // Do not accept invalid words in PASS1.
  if (reject_offset_ <= 0.0f && !is_valid_word) return false;
  if (is_valid_word && is_case_ok) {
    WordSize = LengthOfShortestAlphaRun(*BestChoice);
    WordSize -= stopper_smallword_size;
    if (WordSize < 0)
      WordSize = 0;
    CertaintyThreshold += WordSize * stopper_certainty_per_char;
  }

  if (stopper_debug_level >= 1)
    tprintf("Stopper:  Certainty = %4.1f, Threshold = %4.1f\n",
            BestChoice->certainty(), CertaintyThreshold);

  if (no_dang_ambigs &&
      BestChoice->certainty() > CertaintyThreshold &&
      UniformCertainties(*Choices, *BestChoice)) {
    return true;
  } else {
    if (stopper_debug_level >= 2) {
      tprintf("AcceptableChoice() returned false"
              " (no_dang_ambig:%d cert:%g thresh:%g uniform:%d)\n",
              no_dang_ambigs, BestChoice->certainty(),
              CertaintyThreshold,
              UniformCertainties(*Choices, *BestChoice));
    }
    return false;
  }
}

bool tesseract::Dict::AcceptableResult

(

const WERD_CHOICE &

BestChoice

)

Returns false if the best choice for the current word is questionable and should be tried again on the second pass or should be flagged to the user.

Definition at line 254 of file stopper.cpp.

                                                         {
  float CertaintyThreshold = stopper_nondict_certainty_base - reject_offset_;
  int WordSize;

  if (stopper_debug_level >= 1) {
    tprintf("\nRejecter: %s (word=%c, case=%c, unambig=%c)\n",
            BestChoice.debug_string().string(),
            (valid_word(BestChoice) ? 'y' : 'n'),
            (case_ok(BestChoice, getUnicharset()) ? 'y' : 'n'),
            ((list_rest (best_choices_) != NIL_LIST) ? 'n' : 'y'));
  }

  if (BestChoice.length() == 0 || CurrentWordAmbig())
    return false;
  if (BestChoice.fragment_mark()) {
    if (stopper_debug_level >= 1) {
      cprintf("AcceptableResult(): a choice with fragments beats BestChoice\n");
    }
    return false;
  }
  if (valid_word(BestChoice) && case_ok(BestChoice, getUnicharset())) {
    WordSize = LengthOfShortestAlphaRun(BestChoice);
    WordSize -= stopper_smallword_size;
    if (WordSize < 0)
      WordSize = 0;
    CertaintyThreshold += WordSize * stopper_certainty_per_char;
  }

  if (stopper_debug_level >= 1)
    cprintf ("Rejecter: Certainty = %4.1f, Threshold = %4.1f   ",
      BestChoice.certainty(), CertaintyThreshold);

  if (BestChoice.certainty() > CertaintyThreshold &&
      !stopper_no_acceptable_choices) {
    if (stopper_debug_level >= 1)
      cprintf("ACCEPTED\n");
    return true;
  }
  else {
    if (stopper_debug_level >= 1)
      cprintf("REJECTED\n");
    return false;
  }
}

void tesseract::Dict::add_document_word

(

const WERD_CHOICE &

best_choice

)

Adds a word found on this document to the document specific dictionary.

Definition at line 690 of file dict.cpp.

                                                           {
  // Do not add hyphenated word parts to the document dawg.
  // hyphen_word_ will be non-NULL after the set_hyphen_word() is
  // called when the first part of the hyphenated word is
  // discovered and while the second part of the word is recognized.
  // hyphen_word_ is cleared in cc_recg() before the next word on
  // the line is recognized.
  if (hyphen_word_) return;

  char filename[CHARS_PER_LINE];
  FILE *doc_word_file;
  int stringlen = best_choice.length();

  if (!doc_dict_enable || valid_word(best_choice) ||
      CurrentWordAmbig() || stringlen < 2)
    return;

  // Discard words that contain >= kDocDictMaxRepChars repeating unichars.
  if (best_choice.length() >= kDocDictMaxRepChars) {
    int num_rep_chars = 1;
    UNICHAR_ID uch_id = best_choice.unichar_id(0);
    for (int i = 1; i < best_choice.length(); ++i) {
      if (best_choice.unichar_id(i) != uch_id) {
        num_rep_chars = 1;
        uch_id = best_choice.unichar_id(i);
      } else {
        ++num_rep_chars;
        if (num_rep_chars == kDocDictMaxRepChars) return;
      }
    }
  }

  if (best_choice.certainty() < doc_dict_certainty_threshold ||
      stringlen == 2) {
    if (best_choice.certainty() < doc_dict_pending_threshold)
      return;

    if (!pending_words_->word_in_dawg(best_choice)) {
      if (stringlen > 2 ||
          (stringlen == 2 &&
           getUnicharset().get_isupper(best_choice.unichar_id(0)) &&
           getUnicharset().get_isupper(best_choice.unichar_id(1)))) {
        pending_words_->add_word_to_dawg(best_choice);
      }
      return;
    }
  }

  if (save_doc_words) {
    strcpy(filename, getImage()->getCCUtil()->imagefile.string());
    strcat(filename, ".doc");
    doc_word_file = open_file (filename, "a");
    fprintf(doc_word_file, "%s\n",
            best_choice.debug_string().string());
    fclose(doc_word_file);
  }
  document_words_->add_word_to_dawg(best_choice);
}

void tesseract::Dict::AddNewChunk	(	VIABLE_CHOICE	Choice,
		int	Blob
	)

Increments the chunk count of the character in Choice which corresponds to Blob (index of the blob being split).

Definition at line 788 of file stopper.cpp.

                                                     {
  int i, LastChunk;
  for (i = 0, LastChunk = 0; i < Choice->Length; i++) {
    LastChunk += Choice->Blob[i].NumChunks;
    if (Blob < LastChunk) {
      (Choice->Blob[i].NumChunks)++;
      return;
    }
  }
  cprintf ("AddNewChunk failed:Choice->Length=%d, LastChunk=%d, Blob=%d\n",
           Choice->Length, LastChunk, Blob);
  assert(false);  // this should never get executed
}

void tesseract::Dict::adjust_non_word ( WERD_CHOICE *  word, float *  certainty_array, const BLOB_CHOICE_LIST_VECTOR *  char_choices, bool  debug  )

inline

Definition at line 719 of file dict.h.

                                   {
    adjust_word(word, certainty_array, char_choices, true, 0.0f, debug);
  }

void tesseract::Dict::adjust_word	(	WERD_CHOICE *	word,
		float *	certainty_array,
		const BLOB_CHOICE_LIST_VECTOR *	char_choices,
		bool	nonword,
		float	additional_adjust,
		bool	debug
	)

Adjusts the rating of the given word.

Definition at line 749 of file dict.cpp.

                                   {
  bool is_han = (char_choices != NULL &&
                 getUnicharset().han_sid() != getUnicharset().null_sid() &&
                 get_top_word_script(*char_choices, getUnicharset()) ==
                 getUnicharset().han_sid());
  bool case_is_ok = (is_han || case_ok(*word, getUnicharset()));
  bool punc_is_ok = (is_han || !nonword || valid_punctuation(*word));

  float adjust_factor = additional_adjust;
  float new_rating = word->rating();
  if (debug) {
    tprintf("%sWord: %s %4.2f ", nonword ? "Non-" : "",
            word->debug_string().string(), word->rating());
  }
  new_rating += kRatingPad;
  if (nonword) {  // non-dictionary word
    if (case_is_ok && punc_is_ok) {
      adjust_factor += segment_penalty_dict_nonword;
      new_rating *= adjust_factor;
      if (debug) tprintf(", W");
    } else {
      adjust_factor += segment_penalty_garbage;
      new_rating *= adjust_factor;
      if (debug) {
        if (!case_is_ok) tprintf(", C");
        if (!punc_is_ok) tprintf(", P");
      }
    }
  } else {  // dictionary word
    if (case_is_ok) {
      if (!is_han && freq_dawg_ != NULL && freq_dawg_->word_in_dawg(*word)) {
        word->set_permuter(FREQ_DAWG_PERM);
        adjust_factor += segment_penalty_dict_frequent_word;
        new_rating *= adjust_factor;
        if (debug) tprintf(", F");
      } else {
        adjust_factor += segment_penalty_dict_case_ok;
        new_rating *= adjust_factor;
        if (debug) tprintf(", ");
      }
    } else {
      adjust_factor += segment_penalty_dict_case_bad;
      new_rating *= adjust_factor;
      if (debug) tprintf(", C");
    }
  }
  new_rating -= kRatingPad;
  word->set_rating(new_rating);
  if (debug) tprintf(" %4.2f --> %4.2f\n", adjust_factor, new_rating);
  LogNewChoice(adjust_factor, certainty_array, false, word,
               *char_choices);
}

void tesseract::Dict::adjust_word ( WERD_CHOICE *  word, float *  certainty_array, const BLOB_CHOICE_LIST_VECTOR *  char_choices, bool  debug  )

inline

Definition at line 713 of file dict.h.

                               {
    adjust_word(word, certainty_array, char_choices, false, 0.0f,
                debug);
  }

bool tesseract::Dict::AlternativeChoicesWorseThan

(

FLOAT32

Threshold

)

Returns true if there are no alternative choices for the current word or if all alternatives have an adjust factor worse than Threshold.

Definition at line 299 of file stopper.cpp.

                                                        {
  LIST Alternatives;
  VIABLE_CHOICE Choice;
  Alternatives = list_rest (best_choices_);
  iterate(Alternatives) {
    Choice = (VIABLE_CHOICE) first_node (Alternatives);
    if (Choice->AdjustFactor <= Threshold)
      return false;
  }
  return true;
}

bool tesseract::Dict::ambigs_mode ( float  rating_limit )

inline

Returns true if we are operating in ambigs mode.

Definition at line 180 of file dict.h.

                                              {
    return rating_limit <= 0.0;
  }

void tesseract::Dict::append_choices	(	const char *	debug,
		const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		const BLOB_CHOICE &	blob_choice,
		int	char_choice_index,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		WERD_CHOICE *	word,
		float	certainties[],
		float *	limit,
		WERD_CHOICE *	best_choice,
		int *	attempts_left,
		void *	more_args
	)

append_choices

Checks to see whether or not the next choice is worth appending to the word being generated. If so then keeps going deeper into the word.

This function assumes that Dict::go_deeper_fxn_ is set.

Definition at line 1479 of file permute.cpp.

                     {
  int word_ending =
    (char_choice_index == char_choices.length() - 1) ? true : false;

  // Deal with fragments.
  CHAR_FRAGMENT_INFO char_frag_info;
  if (!fragment_state_okay(blob_choice.unichar_id(), blob_choice.rating(),
                           blob_choice.certainty(), prev_char_frag_info, debug,
                           word_ending, &char_frag_info)) {
    return;  // blob_choice must be an invalid fragment
  }
  // Search the next letter if this character is a fragment.
  if (char_frag_info.unichar_id == INVALID_UNICHAR_ID) {
    permute_choices(debug, char_choices, char_choice_index + 1,
                    &char_frag_info, word, certainties, limit,
                    best_choice, attempts_left, more_args);
    return;
  }

  // Add the next unichar.
  float old_rating = word->rating();
  float old_certainty = word->certainty();
  uinT8 old_permuter = word->permuter();
  certainties[word->length()] = char_frag_info.certainty;
  word->append_unichar_id_space_allocated(
      char_frag_info.unichar_id, char_frag_info.num_fragments,
      char_frag_info.rating, char_frag_info.certainty);

  // Explore the next unichar.
  (this->*go_deeper_fxn_)(debug, char_choices, char_choice_index,
                          &char_frag_info, word_ending, word, certainties,
                          limit, best_choice, attempts_left, more_args);

  // Remove the unichar we added to explore other choices in it's place.
  word->remove_last_unichar_id();
  word->set_rating(old_rating);
  word->set_certainty(old_certainty);
  word->set_permuter(old_permuter);
}

int tesseract::Dict::case_ok	(	const WERD_CHOICE &	word,
		const UNICHARSET &	unicharset
	)

Check a string to see if it matches a set of lexical rules.

Definition at line 58 of file context.cpp.

                                                                       {
  int last_state = 0;
  int state = 0;
  int x;
  for (x = 0; x < word.length(); ++x) {
    UNICHAR_ID ch_id = word.unichar_id(x);
    if (unicharset.get_isupper(ch_id))
      state = case_state_table[state][1];
    else if (unicharset.get_islower(ch_id))
      state = case_state_table[state][2];
    else if (unicharset.get_isdigit(ch_id))
      state = case_state_table[state][3];
    else
      state = case_state_table[state][0];
    if (state == -1) return false;
    last_state = state;
  }
  return state != 5; // single lower is bad
}

bool tesseract::Dict::ChoiceAccumEnabled ( )

inline

Definition at line 346 of file dict.h.

{ return keep_word_choices_; }

int tesseract::Dict::ChoiceSameAs	(	const WERD_CHOICE &	WordChoice,
		VIABLE_CHOICE	ViableChoice
	)

Compares the corresponding strings of WordChoice and ViableChoice and returns true if they are the same.

Definition at line 878 of file stopper.cpp.

                                                   {
  return (StringSameAs(WordChoice, ViableChoice));
}

const char * tesseract::Dict::choose_il1	(	const char *	first_char,
		const char *	second_char,
		const char *	third_char,
		const char *	prev_char,
		const char *	next_char,
		const char *	next_next_char
	)

Definition at line 1163 of file permute.cpp.

                                                         {
  inT32 type1;                   //1/I/l type of first choice
  inT32 type2;                   //1/I/l type of second choice
  inT32 type3;                   //1/I/l type of third choice

  int first_char_length = strlen(first_char);
  int prev_char_length = strlen(prev_char);
  int next_char_length = strlen(next_char);
  int next_next_char_length = strlen(next_next_char);

  if (*first_char == 'l' && *second_char != '\0') {
    if (*second_char == 'I'
        && (((prev_char_length != 0 &&
            getUnicharset().get_isupper (prev_char, prev_char_length)) &&
            (next_char_length == 0 ||
             !getUnicharset().get_islower (next_char, next_char_length)) &&
            (next_char_length == 0 ||
             !getUnicharset().get_isdigit (next_char, next_char_length))) ||
            ((next_char_length != 0 &&
             getUnicharset().get_isupper (next_char, next_char_length)) &&
            (prev_char_length == 0 ||
             !getUnicharset().get_islower (prev_char, prev_char_length)) &&
            (prev_char_length == 0 ||
             !getUnicharset().get_isdigit (prev_char, prev_char_length)))))
      first_char = second_char;  //override
    else if (*second_char == '1' || *third_char == '1') {
      if ((next_char_length != 0 &&
           getUnicharset().get_isdigit (next_char, next_char_length)) ||
          (prev_char_length != 0 &&
           getUnicharset().get_isdigit (prev_char, prev_char_length))
          || (*next_char == 'l' &&
          (next_next_char_length != 0 &&
           getUnicharset().get_isdigit (next_next_char,
                                        next_next_char_length)))) {
        first_char = "1";
        first_char_length = 1;
      }
      else if ((prev_char_length == 0 ||
                !getUnicharset().get_islower (prev_char, prev_char_length)) &&
               ((next_char_length == 0 ||
                 !getUnicharset().get_islower (next_char, next_char_length)) ||
                (*next_char == 's' &&
                *next_next_char == 't'))) {
        if (((*prev_char != '\'' && *prev_char != '`') || *next_char != '\0')
            && ((*next_char != '\'' && *next_char != '`')
                || *prev_char != '\0')) {
          first_char = "1";
          first_char_length = 1;
        }
      }
    }
    if (*first_char == 'l' && *next_char != '\0' &&
        (prev_char_length == 0 ||
         !getUnicharset().get_isalpha (prev_char, prev_char_length))) {
      type1 = 2;

      if (*second_char == '1')
        type2 = 0;
      else if (*second_char == 'I')
        type2 = 1;
      else if (*second_char == 'l')
        type2 = 2;
      else
        type2 = type1;

      if (*third_char == '1')
        type3 = 0;
      else if (*third_char == 'I')
        type3 = 1;
      else if (*third_char == 'l')
        type3 = 2;
      else
        type3 = type1;

#if 0
      if (bigram_counts[*next_char][type2] >
      bigram_counts[*next_char][type1]) {
        first_char = second_char;
        type1 = type2;
      }
      if (bigram_counts[*next_char][type3] >
      bigram_counts[*next_char][type1]) {
        first_char = third_char;
      }
#endif
    }
  }
  return first_char;
}

void tesseract::Dict::ClearBestChoiceAccum

(

)

Clears best_choices_ list accumulated by the stopper.

Definition at line 458 of file stopper.cpp.

                                {
  if (best_choices_) destroy_nodes(best_choices_, DeleteViableChoiceStruct);
  best_choices_ = NIL_LIST;
}

bool tesseract::Dict::compound_marker ( UNICHAR_ID  unichar_id )

inline

Definition at line 110 of file dict.h.

                                                     {
    return (unichar_id == getUnicharset().unichar_to_id("-") ||
            unichar_id == getUnicharset().unichar_to_id("/"));
  }

bool tesseract::Dict::ConstraintsOk ( const DawgInfoVector &  constraints, int  word_end, DawgType  current_dawg_type  ) const

inline

At word ending make sure all the recorded constraints are satisfied. Each constraint signifies that we found a beginning pattern in a pattern dawg. Check that this pattern can end here (e.g. if some leading punctuation is found this would ensure that we are not expecting any particular trailing punctuation after the word).

Definition at line 631 of file dict.h.

                                                                            {
    if (!word_end) return true;
    if (current_dawg_type == DAWG_TYPE_PUNCTUATION) return true;
    for (int c = 0; c < constraints.length(); ++c) {
      const DawgInfo &cinfo = constraints[c];
      Dawg *cdawg = dawgs_[cinfo.dawg_index];
      if (!cdawg->end_of_word(cinfo.ref)) {
        if (dawg_debug_level >= 3) {
          tprintf("Constraint [%d, " REFFORMAT "] is not satisfied\n",
                  cinfo.dawg_index, cinfo.ref);
        }
        return false;
      }
    }
    return true;
  }

void tesseract::Dict::copy_hyphen_info ( WERD_CHOICE *  word ) const

inline

If this word is hyphenated copy the base word (the part on the line before) of a hyphenated word into the given word. This function assumes that word is not NULL.

Definition at line 128 of file dict.h.

                                                        {
    if (this->hyphenated()) {
      *word = *hyphen_word_;
      if (hyphen_debug_level) word->print("copy_hyphen_info: ");
    }
  }

FLOAT32 tesseract::Dict::CurrentBestChoiceAdjustFactor

(

)

Returns the adjustment factor for the best choice for the current word.

Definition at line 316 of file stopper.cpp.

                                            {
  VIABLE_CHOICE BestChoice;
  if (best_choices_ == NIL_LIST)
    return (MAX_FLOAT32);
  BestChoice = (VIABLE_CHOICE) first_node (best_choices_);
  return (BestChoice->AdjustFactor);
}

bool tesseract::Dict::CurrentBestChoiceIs

(

const WERD_CHOICE &

WordChoice

)

Returns true if WordChoice is the same as the current best choice.

Definition at line 311 of file stopper.cpp.

                                                            {
  return (best_choices_ != NIL_LIST &&
          StringSameAs(WordChoice, (VIABLE_CHOICE)first_node(best_choices_)));
}

bool tesseract::Dict::CurrentWordAmbig

(

)

Returns true if there are multiple good choices for the current word.

Definition at line 325 of file stopper.cpp.

                            {
  return (list_rest (best_choices_) != NIL_LIST);
}

WERD_CHOICE * tesseract::Dict::dawg_permute_and_select	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit,
		int	sought_word_length,
		int	start_char_choice_index
	)

Recursively explore all the possible character combinations in the given char_choices. Use go_deeper_dawg_fxn() to explore all the dawgs in the dawgs_ vector in parallel and discard invalid words.

Allocate and return a WERD_CHOICE with the best valid word found.

dawg_permute_and_select

Recursively explore all the possible character combinations in the given char_choices. Use go_deeper_dawg_fxn() to search all the dawgs in the dawgs_ vector in parallel and discard invalid words.

If sought_word_length is not kAnyWordLength, the function only searches for a valid word formed by the given char_choices in one fixed length dawg (that contains words of length sought_word_length) starting at the start_char_choice_index.

Allocate and return a WERD_CHOICE with the best valid word found.

Definition at line 263 of file permdawg.cpp.

                                                         {
  WERD_CHOICE *best_choice = new WERD_CHOICE(&getUnicharset());
  best_choice->make_bad();
  best_choice->set_rating(rating_limit);
  if (char_choices.length() == 0) return best_choice;
  DawgInfoVector *active_dawgs = new DawgInfoVector[char_choices.length() + 1];
  DawgInfoVector *constraints =  new DawgInfoVector[char_choices.length() + 1];
  init_active_dawgs(sought_word_length, &(active_dawgs[0]),
                    ambigs_mode(rating_limit));
  init_constraints(&(constraints[0]));
  int end_char_choice_index = (sought_word_length == kAnyWordLength) ?
    char_choices.length()-1 : start_char_choice_index+sought_word_length-1;
  // Need to skip accumulating word choices if we are only searching a part of
  // the word (e.g. for the phrase search in non-space delimited languages).
  // Also need to skip accumulating choices if char_choices are expanded
  // with ambiguities.
  bool re_enable_choice_accum = ChoiceAccumEnabled();
  if (sought_word_length != kAnyWordLength ||
      ambigs_mode(rating_limit)) DisableChoiceAccum();
  DawgArgs dawg_args(&(active_dawgs[0]), &(constraints[0]),
                     &(active_dawgs[1]), &(constraints[1]),
                     (segment_penalty_dict_case_bad /
                      segment_penalty_dict_case_ok),
                     NO_PERM, sought_word_length, end_char_choice_index);
  WERD_CHOICE word(&getUnicharset(), MAX_WERD_LENGTH);
  copy_hyphen_info(&word);
  // Discard rating and certainty of the hyphen base (if any).
  word.set_rating(0.0);
  word.set_certainty(0.0);
  if (word.length() + char_choices.length() > MAX_WERD_LENGTH) {
    delete[] active_dawgs;
    delete[] constraints;
    return best_choice;  // the word is too long to permute
  }
  float certainties[MAX_WERD_LENGTH];
  this->go_deeper_fxn_ = &tesseract::Dict::go_deeper_dawg_fxn;
  int attempts_left = max_permuter_attempts;
  permute_choices((permute_debug && dawg_debug_level) ?
                  "permute_dawg_debug" : NULL,
                  char_choices, start_char_choice_index, NULL, &word,
                  certainties, &rating_limit, best_choice, &attempts_left,
                  &dawg_args);
  delete[] active_dawgs;
  delete[] constraints;
  if (re_enable_choice_accum) EnableChoiceAccum();
  return best_choice;
}

WERD_CHOICE* tesseract::Dict::dawg_permute_and_select ( const BLOB_CHOICE_LIST_VECTOR &  char_choices, float  rating_limit  )

inline

Definition at line 191 of file dict.h.

                                                                       {
    return dawg_permute_and_select(char_choices, rating_limit,
                                   kAnyWordLength, 0);
  }

void tesseract::Dict::DebugWordChoices

(

)

Prints the current choices for this word to stdout.

Definition at line 330 of file stopper.cpp.

                            {
  LIST Choices;
  int i;
  char LabelString[80];
  VIABLE_CHOICE VChoice = (VIABLE_CHOICE)first_node(best_choices_);
  bool force_debug =
    fragments_debug && VChoice != NULL && VChoice->ComposedFromCharFragments;

  if (stopper_debug_level >= 1 || force_debug ||
  (((STRING)word_to_debug).length() > 0 && best_choices_ &&
       StringSameAs(word_to_debug.string(), word_to_debug_lengths.string(),
                    (VIABLE_CHOICE)first_node(best_choices_)))) {
    if (best_raw_choice_)
      PrintViableChoice(stderr, "\nBest Raw Choice:   ", best_raw_choice_);

    i = 1;
    Choices = best_choices_;
    if (Choices)
      cprintf("\nBest Cooked Choices:\n");
    iterate(Choices) {
      sprintf(LabelString, "Cooked Choice #%d:  ", i);
      PrintViableChoice(stderr, LabelString,
                        (VIABLE_CHOICE)first_node(Choices));
      i++;
    }
  }
}

int tesseract::Dict::def_letter_is_okay	(	void *	void_dawg_args,
		UNICHAR_ID	unichar_id,
		bool	word_end
	)		const

Returns the maximal permuter code (from ccstruct/ratngs.h) if in light of the current state the letter at word_index in the given word is allowed according to at least one of the dawgs in dawgs_, otherwise returns NO_PERM.

The state is described by void_dawg_args, which are interpreted as DawgArgs and contain two relevant input vectors: active_dawgs and constraints. Each entry in the active_dawgs vector contains an index into the dawgs_ vector and an EDGE_REF that indicates the last edge followed in the dawg. Each entry in the constraints vector contains an index into the dawgs_ vector and an EDGE_REF that indicates an edge in a pattern dawg followed to match a pattern. Currently constraints are used to save the state of punctuation dawgs after leading punctuation was found.

Input: At word_index 0 dawg_args->active_dawgs should contain an entry for each dawg whose type has a bit set in kBeginningDawgsType, dawg_args->constraints should be empty. EDGE_REFs in active_dawgs and constraints vectors should be initialized to NO_EDGE. If hyphen state needs to be applied, initial dawg_args->active_dawgs and dawg_args->constrains can be copied from the saved hyphen state (maintained by Dict). For word_index > 0 the corresponding state (active_dawgs and constraints) can be obtained from dawg_args->updated_* passed to def_letter_is_okay for word_index-1. Note: the function assumes that active_dags, constraints and updated_* member variables of dawg_args are not NULL.

Output: The function fills in dawg_args->updated_active_dawgs vector with the entries for dawgs that contain the word up to the letter at word_index. The new constraints (if any) are added to dawg_args->updated_constraints, the constraints from dawg_args->constraints are also copied into it.

Detailed description: In order to determine whether the word is still valid after considering all the letters up to the one at word_index the following is done for each entry in dawg_args->active_dawgs:

• next starting node is obtained from entry.ref and edge_char_of() is called to obtain the next edge
• if a valid edge is found, the function returns the updated permuter code true and an entry [entry.dawg_index, edge] is inserted in dawg_args->updated_active_dawgs otherwise:
  • if we are dealing with dawg of type DAWG_TYPE_PUNCTUATION, edge_char_of() is called again, but now with kPatternUnicharID as unichar_id; if a valid edge is found it is recorded in dawg_args->updated_constraints
  • the function checks whether the word can end with the previous letter
  • each successor of the dawg (e.g. dawgs with type DAWG_TYPE_WORD could be successors to dawgs with type DAWG_TYPE_PUNCTUATION; the successors are defined by successors_ vector) is explored and if a letter is found in the successor dawg, a new entry is inserted into dawg_args->updated_active_dawgs with EDGE_REF being either NO_EDGE or an EDGE_REF recorded in constraints vector for the corresponding dawg index

Definition at line 380 of file dict.cpp.

                                                  {
  DawgArgs *dawg_args = reinterpret_cast<DawgArgs*>(void_dawg_args);

  if (dawg_debug_level >= 3) {
    tprintf("def_letter_is_okay: current unichar=%s word_end=%d"
            " num active dawgs=%d num constraints=%d\n",
            getUnicharset().debug_str(unichar_id).string(), word_end,
            dawg_args->active_dawgs->length(),
            dawg_args->constraints->length());
  }

  // Do not accept words that contain kPatternUnicharID.
  // (otherwise pattern dawgs would not function correctly).
  // Do not accept words containing INVALID_UNICHAR_IDs.
  if (unichar_id == Dawg::kPatternUnicharID ||
      unichar_id == INVALID_UNICHAR_ID) {
    dawg_args->permuter = NO_PERM;
    return NO_PERM;
  }

  // Initialization.
  PermuterType curr_perm = NO_PERM;
  dawg_args->updated_active_dawgs->clear();
  const DawgInfoVector &constraints = *(dawg_args->constraints);
  *dawg_args->updated_constraints = constraints;

  // Go over the active_dawgs vector and insert DawgInfo records with the
  // updated ref (an edge with the corresponding unichar id) into
  // dawg_args->updated_active_dawgs.
  for (int a = 0; a < dawg_args->active_dawgs->length(); ++a) {
    const DawgInfo &info = (*dawg_args->active_dawgs)[a];
    const Dawg *dawg = dawgs_[info.dawg_index];
    // dawg_unichar_id will contain the literal unichar_id to be found in the
    // dawgs (e.g. didgit pattern if unichar_id is a digit and dawg contains
    // number patterns, word pattern if dawg is a puncutation dawg and we
    // reached an end of beginning puntuation pattern, etc).
    UNICHAR_ID dawg_unichar_id = unichar_id;

    // If we are dealing with the pattern dawg, look up all the
    // possible edges, not only for the exact unichar_id, but also
    // for all its character classes (alpha, digit, etc).
    if (dawg->type() == DAWG_TYPE_PATTERN) {
      ProcessPatternEdges(dawg, info, dawg_unichar_id, word_end,
                          dawg_args, &curr_perm);
      // There can't be any successors to dawg that is of type
      // DAWG_TYPE_PATTERN, so we are done examining this DawgInfo.
      continue;
    }

    // The number dawg generalizes all digits to be kPatternUnicharID,
    // so try to match kPatternUnicharID if the current unichar is a digit.
    if (dawg->type() == DAWG_TYPE_NUMBER &&
        getUnicharset().get_isdigit(dawg_unichar_id)) {
      dawg_unichar_id = Dawg::kPatternUnicharID;
    }

    // Find the edge out of the node for the dawg_unichar_id.
    NODE_REF node = GetStartingNode(dawg, info.ref);
    EDGE_REF edge = (node != NO_EDGE) ?
      dawg->edge_char_of(node, dawg_unichar_id, word_end) : NO_EDGE;

    if (dawg_debug_level >= 3) {
      tprintf("Active dawg: [%d, " REFFORMAT "] edge=" REFFORMAT "\n",
              info.dawg_index, node, edge);
    }

    if (edge != NO_EDGE) {  // the unichar was found in the current dawg
      if (ConstraintsOk(*(dawg_args->updated_constraints),
                        word_end, dawg->type())) {
        if (dawg_debug_level >=3) {
          tprintf("Letter found in dawg %d\n", info.dawg_index);
        }
        if (dawg->permuter() > curr_perm) curr_perm = dawg->permuter();
        dawg_args->updated_active_dawgs->add_unique(
            DawgInfo(info.dawg_index, edge), dawg_debug_level > 0,
            "Append current dawg to updated active dawgs: ");
      }
    } else if (dawg_args->sought_word_length == kAnyWordLength) {
      // The unichar was not found in the current dawg.
      // Explore the successor dawgs (but only if we are not
      // just searching one dawg with a fixed word length).

      // Handle leading/trailing punctuation dawgs that denote a word pattern
      // as an edge with kPatternUnicharID. If such an edge is found we add a
      // constraint denoting the state of the dawg before the word pattern.
      // This constraint will be applied later when this dawg is found among
      // successor dawgs as well potentially at the end of the word.
      if (dawg->type() == DAWG_TYPE_PUNCTUATION) {
        edge = dawg->edge_char_of(node, Dawg::kPatternUnicharID, word_end);
        if (edge != NO_EDGE) {
          dawg_args->updated_constraints->add_unique(
              DawgInfo(info.dawg_index, edge), dawg_debug_level > 0,
              "Recording constraint: ");
        } else {
          // Do not explore successors of this dawg, since this
          // must be invalid leading or trailing punctuation.
          if (dawg_debug_level >= 3) {
            tprintf("Invalid punctuation from dawg %d\n", info.dawg_index);
          }
          continue;
        }
      }

      if (info.ref == NO_EDGE) {
        if (dawg_debug_level >= 3) {
          tprintf("No letters matched in dawg %d\n", info.dawg_index);
        }
        continue;
      }

      // Discard the dawg if the pattern can not end at previous letter.
      if (edge == NO_EDGE &&  // previous part is not leading punctuation
          !dawg->end_of_word(info.ref)) {
        if (dawg_debug_level >= 3) {
          tprintf("No valid pattern end in dawg %d\n", info.dawg_index);
        }
        continue;
      }

      // Look for the unichar in each of this dawg's successors
      // and append those in which it is found to active_dawgs.
      const SuccessorList &slist = *(successors_[info.dawg_index]);
      for (int s = 0; s < slist.length(); ++s) {
        int sdawg_index = slist[s];
        const Dawg *sdawg = dawgs_[sdawg_index];
        NODE_REF snode = 0;
        // Apply constraints to the successor dawg.
        for (int c = 0; c < constraints.length(); ++c) {
          // If the successor dawg is described in the constraints change
          // the start ref from 0 to the one recorded as the constraint.
          const DawgInfo &cinfo = constraints[c];
          if (cinfo.dawg_index == sdawg_index) {
            snode = sdawg->next_node(cinfo.ref);
            // Make sure we do not search the successor dawg if after
            // applying the saved constraint we are at the end of the word.
            if (snode == 0) snode = NO_EDGE;
            if (dawg_debug_level >= 3) {
               tprintf("Applying constraint [%d, " REFFORMAT "]\n",
                       sdawg_index, snode);
            }
          }
        }
        // Look for the letter in this successor dawg.
        EDGE_REF sedge = sdawg->edge_char_of(snode, unichar_id, word_end);
        // If we found the letter append sdawg to the active_dawgs list.
        if (sedge != NO_EDGE &&
            ConstraintsOk(*(dawg_args->updated_constraints), word_end,
                          dawgs_[sdawg_index]->type())) {
          if (dawg_debug_level >= 3) {
            tprintf("Letter found in the successor dawg %d\n", sdawg_index);
          }
          if (sdawg->permuter() > curr_perm) curr_perm = sdawg->permuter();
          if (sdawg->next_node(sedge) != 0) {  // if not word end
            dawg_args->updated_active_dawgs->add_unique(
              DawgInfo(sdawg_index, sedge), dawg_debug_level > 0,
              "Append successor to updated active dawgs: ");
          }
        }
      }  // end successors loop
    }  // end if/else
  }  // end for
  // Update dawg_args->permuter if it used to be NO_PERM or became NO_PERM
  // or if we found the current letter in a non-punctuation dawg. This
  // allows preserving information on which dawg the "core" word came from.
  // Keep the old value of dawg_args->permuter if it is COMPOUND_PERM.
  if (dawg_args->permuter == NO_PERM || curr_perm == NO_PERM ||
      (curr_perm != PUNC_PERM && dawg_args->permuter != COMPOUND_PERM)) {
    dawg_args->permuter = curr_perm;
  }
  return dawg_args->permuter;
}

double tesseract::Dict::def_probability_in_context ( const char *  lang, const char *  context, int  context_bytes, const char *  character, int  character_bytes  )

inline

Default (no-op) implementation of probability in context function.

Definition at line 587 of file dict.h.

                                                  {
    (void) context;
    (void) context_bytes;
    (void) character;
    (void) character_bytes;
    return 0.0;
  }

void tesseract::Dict::DisableChoiceAccum ( )

inline

Definition at line 344 of file dict.h.

{ keep_word_choices_ = false; }

void tesseract::Dict::EnableChoiceAccum ( )

inline

Definition at line 345 of file dict.h.

{ keep_word_choices_ = true; }

void tesseract::Dict::End

(

)

Definition at line 335 of file dict.cpp.

               {
  if (dawgs_.length() == 0)
    return;  // Not safe to call twice.
  dawgs_.delete_data_pointers();
  successors_.delete_data_pointers();
  dawgs_.clear();
  delete bigram_dawg_;
  successors_.clear();
  document_words_ = NULL;
  max_fixed_length_dawgs_wdlen_ = -1;
  if (pending_words_ != NULL) {
    delete pending_words_;
    pending_words_ = NULL;
  }
}

void tesseract::Dict::end_permute

(

)

void tesseract::Dict::EndDangerousAmbigs

(

)

Definition at line 778 of file stopper.cpp.

{}

void tesseract::Dict::FillViableChoice	(	const WERD_CHOICE &	WordChoice,
		FLOAT32	AdjustFactor,
		const float	Certainties[],
		VIABLE_CHOICE	ViableChoice
	)

Fill ViableChoice with information from WordChoice, AChoice, AdjustFactor, and Certainties.

Definition at line 943 of file stopper.cpp.

                                                        {
  ViableChoice->Init(WordChoice, current_segmentation_, Certainties,
                     AdjustFactor);

}

void tesseract::Dict::FilterWordChoices

(

)

Removes from best_choices_ all choices which are not within a reasonable range of the best choice.

Definition at line 375 of file stopper.cpp.

                             {
  EXPANDED_CHOICE BestChoice;

  if (best_choices_ == NIL_LIST || second_node (best_choices_) == NIL_LIST)
    return;

  // Compute certainties and class for each chunk in best choice.
  VIABLE_CHOICE_STRUCT *best_choice =
      (VIABLE_CHOICE_STRUCT *)first_node(best_choices_);
  ExpandChoice(best_choice, &BestChoice);
  if (stopper_debug_level >= 2)
    PrintViableChoice(stderr, "\nFiltering against best choice: ", best_choice);
  TessResultCallback2<int, void*, void*>* is_bad =
      NewPermanentTessCallback(this, &Dict::FreeBadChoice);
  set_rest(best_choices_, delete_d(list_rest(best_choices_),
                                   &BestChoice, is_bad));
  delete is_bad;
}

void tesseract::Dict::FindClassifierErrors	(	FLOAT32	MinRating,
		FLOAT32	MaxRating,
		FLOAT32	RatingMargin,
		FLOAT32	Thresholds[]
	)

Compares the best choice for the current word to the best raw choice to determine which characters were classified incorrectly by the classifier. Then places a separate threshold into Thresholds for each character in the word. If the classifier was correct, MaxRating is placed into Thresholds. If the classifier was incorrect, the avg. match rating (error percentage) of the classifier's incorrect choice minus some margin is placed into thresholds.This can then be used by the caller to try to create a new template for the desired class that will classify the character with a rating better than the threshold value. The match rating placed into Thresholds is never allowed to be below MinRating in order to prevent trying to make overly tight templates. MinRating limits how tight to make a template. MaxRating limits how loose to make a template. RatingMargin denotes the amount of margin to put in template.

Definition at line 394 of file stopper.cpp.

                                                      {
  EXPANDED_CHOICE BestRaw;
  VIABLE_CHOICE Choice;
  int i, j, Chunk;
  FLOAT32 AvgRating;
  int NumErrorChunks;

  assert (best_choices_ != NIL_LIST);
  assert (best_raw_choice_ != NULL);

  ExpandChoice(best_raw_choice_, &BestRaw);
  Choice = (VIABLE_CHOICE) first_node (best_choices_);

  for (i = 0, Chunk = 0; i < Choice->Length; i++, Thresholds++) {
    AvgRating = 0.0;
    NumErrorChunks = 0;

    for (j = 0; j < Choice->Blob[i].NumChunks; j++, Chunk++) {
      if (Choice->Blob[i].Class != BestRaw.ChunkClass[Chunk]) {
        AvgRating += BestRaw.ChunkCertainty[Chunk];
        NumErrorChunks++;
      }
    }

    if (NumErrorChunks > 0) {
      AvgRating /= NumErrorChunks;
      *Thresholds = (AvgRating / -certainty_scale) * (1.0 - RatingMargin);
    }
    else
      *Thresholds = MaxRating;

    if (*Thresholds > MaxRating)
      *Thresholds = MaxRating;
    if (*Thresholds < MinRating)
      *Thresholds = MinRating;
  }
}

bool tesseract::Dict::fragment_state_okay	(	UNICHAR_ID	curr_unichar_id,
		float	curr_rating,
		float	curr_certainty,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		const char *	debug,
		int	word_ending,
		CHAR_FRAGMENT_INFO *	char_frag_info
	)

Semi-generic functions used by multiple permuters.

Definition at line 1283 of file permute.cpp.

                                                                   {
  const CHAR_FRAGMENT *this_fragment =
    getUnicharset().get_fragment(curr_unichar_id);
  const CHAR_FRAGMENT *prev_fragment =
    prev_char_frag_info != NULL ? prev_char_frag_info->fragment : NULL;

  // Print debug info for fragments.
  if (debug && (prev_fragment || this_fragment)) {
    cprintf("%s check fragments: choice=%s word_ending=%d\n", debug,
            getUnicharset().debug_str(curr_unichar_id).string(),
            word_ending);
    if (prev_fragment) {
      cprintf("prev_fragment %s\n", prev_fragment->to_string().string());
    }
    if (this_fragment) {
      cprintf("this_fragment %s\n", this_fragment->to_string().string());
    }
  }

  char_frag_info->unichar_id = curr_unichar_id;
  char_frag_info->fragment = this_fragment;
  char_frag_info->rating = curr_rating;
  char_frag_info->certainty = curr_certainty;
  char_frag_info->num_fragments = 1;
  if (prev_fragment && !this_fragment) {
    if (debug) tprintf("Skip choice with incomplete fragment\n");
    return false;
  }
  if (this_fragment) {
    // We are dealing with a fragment.
    char_frag_info->unichar_id = INVALID_UNICHAR_ID;
    if (prev_fragment) {
      if (!this_fragment->is_continuation_of(prev_fragment)) {
        if (debug) tprintf("Non-matching fragment piece\n");
        return false;
      }
      if (this_fragment->is_ending()) {
        char_frag_info->unichar_id =
          getUnicharset().unichar_to_id(this_fragment->get_unichar());
        char_frag_info->fragment = NULL;
        if (debug) {
          tprintf("Built character %s from fragments\n",
                  getUnicharset().debug_str(
                      char_frag_info->unichar_id).string());
        }
      } else {
        if (debug) tprintf("Record fragment continuation\n");
        char_frag_info->fragment = this_fragment;
      }
      // Update certainty and rating.
      char_frag_info->rating =
        prev_char_frag_info->rating + curr_rating;
      char_frag_info->num_fragments = prev_char_frag_info->num_fragments + 1;
      char_frag_info->certainty =
        MIN(curr_certainty, prev_char_frag_info->certainty);
    } else {
      if (this_fragment->is_beginning()) {
        if (debug) cprintf("Record fragment beginning\n");
      } else {
        if (debug) {
          tprintf("Non-starting fragment piece with no prev_fragment\n");
        }
        return false;
      }
    }
  }
  if (word_ending && char_frag_info->fragment) {
    if (debug) tprintf("Word can not end with a fragment\n");
    return false;
  }
  return true;
}

int tesseract::Dict::FreeBadChoice	(	void *	item1,
		void *	item2
	)

Definition at line 167 of file stopper.cpp.

                 {  // EXPANDED_CHOICE *BestChoice
  int i, j, Chunk;
  FLOAT32 Threshold;
  VIABLE_CHOICE Choice = reinterpret_cast<VIABLE_CHOICE>(item1);
  EXPANDED_CHOICE *BestChoice = reinterpret_cast<EXPANDED_CHOICE *>(item2);
  Threshold = StopperAmbigThreshold(BestChoice->Choice->AdjustFactor,
                                    Choice->AdjustFactor);
  for (i = 0, Chunk = 0; i < Choice->Length; i++) {
    for (j = 0; j < Choice->Blob[i].NumChunks; j++, Chunk++) {
      if (Choice->Blob[i].Class != BestChoice->ChunkClass[Chunk] &&
          Choice->Blob[i].Certainty - BestChoice->ChunkCertainty[Chunk] <
          Threshold) {
        if (stopper_debug_level >= 2)
          PrintViableChoice(stderr, "\nDiscarding bad choice:  ", Choice);
        delete Choice;
        return true;
      }
    }
  }
  return false;
}

WERD_CHOICE * tesseract::Dict::get_top_choice_word

(

const BLOB_CHOICE_LIST_VECTOR &

char_choices

)

Return the top choice for each character as the choice for the word.

Definition at line 908 of file permute.cpp.

                                                 {
  WERD_CHOICE *top_word = new WERD_CHOICE(&getUnicharset(), MAX_PERM_LENGTH);
  float certainties[MAX_PERM_LENGTH];
  top_word->set_permuter(TOP_CHOICE_PERM);
  for (int x = 0; x < char_choices.length(); x++) {
    BLOB_CHOICE_IT blob_choice_it;
    blob_choice_it.set_to_list(char_choices.get(x));
    BLOB_CHOICE *top_choice = blob_choice_it.data();
    top_word->append_unichar_id_space_allocated(top_choice->unichar_id(), 1,
                                                top_choice->rating(),
                                                top_choice->certainty());
    certainties[x] = top_choice->certainty();
  }
  LogNewChoice(1.0, certainties, true, top_word, char_choices);
  return top_word;
}

int tesseract::Dict::get_top_word_script	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		const UNICHARSET &	unicharset
	)

Definition at line 908 of file dict.cpp.

                                                            {
  int max_script = unicharset.get_script_table_size();
  int *sid = new int[max_script];
  int x;
  for (x = 0; x < max_script; x++) sid[x] = 0;
  for (x = 0; x < char_choices.length(); ++x) {
    BLOB_CHOICE_IT blob_choice_it(char_choices.get(x));
    sid[blob_choice_it.data()->script_id()]++;
  }
  if (unicharset.han_sid() != unicharset.null_sid()) {
    // Add the Hiragana & Katakana counts to Han and zero them out.
    if (unicharset.hiragana_sid() != unicharset.null_sid()) {
      sid[unicharset.han_sid()] += sid[unicharset.hiragana_sid()];
      sid[unicharset.hiragana_sid()] = 0;
    }
    if (unicharset.katakana_sid() != unicharset.null_sid()) {
      sid[unicharset.han_sid()] += sid[unicharset.katakana_sid()];
      sid[unicharset.katakana_sid()] = 0;
    }
  }
  // Note that high script ID overrides lower one on a tie, thus biasing
  // towards non-Common script (if sorted that way in unicharset file).
  int max_sid = 0;
  for (x = 1; x < max_script; x++)
    if (sid[x] >= sid[max_sid]) max_sid = x;
  if (sid[max_sid] < char_choices.length() / 2)
    max_sid = unicharset.null_sid();
  delete[] sid;
  return max_sid;
}

const LIST& tesseract::Dict::getBestChoices ( )

inline

Definition at line 730 of file dict.h.

{ return best_choices_; }

const Dawg* tesseract::Dict::GetDawg ( int  index ) const

inline

Return i-th dawg pointer recorded in the dawgs_ vector.

Definition at line 605 of file dict.h.

{ return dawgs_[index]; }

const Dawg* tesseract::Dict::GetFixedLengthDawg ( int  word_length ) const

inline

Return the pointer to the Dawg that contains words of length word_length.

Definition at line 611 of file dict.h.

                                                               {
    if (word_length > max_fixed_length_dawgs_wdlen_) return NULL;
    assert(dawgs_.size() > word_length);
    return dawgs_[word_length];
  }

const Image* tesseract::Dict::getImage ( ) const

inline

Definition at line 94 of file dict.h.

                                {
    return image_ptr_;
  }

Image* tesseract::Dict::getImage ( )

inline

Definition at line 97 of file dict.h.

                    {
    return image_ptr_;
  }

const int tesseract::Dict::GetMaxFixedLengthDawgIndex ( ) const

inline

Definition at line 616 of file dict.h.

                                                      {
    return max_fixed_length_dawgs_wdlen_;
  }

const Dawg* tesseract::Dict::GetPuncDawg ( ) const

inline

Return the points to the punctuation dawg.

Definition at line 607 of file dict.h.

{ return punc_dawg_; }

static NODE_REF tesseract::Dict::GetStartingNode ( const Dawg *  dawg, EDGE_REF  edge_ref  )

inlinestatic

Returns the appropriate next node given the EDGE_REF.

Definition at line 620 of file dict.h.

                                                                              {
    if (edge_ref == NO_EDGE) return 0;  // beginning to explore the dawg
    NODE_REF node = dawg->next_node(edge_ref);
    if (node == 0) node = NO_EDGE;  // end of word
    return node;
  }

const Dawg* tesseract::Dict::GetUnambigDawg ( ) const

inline

Return the points to the unambiguous words dawg.

Definition at line 609 of file dict.h.

{ return unambig_dawg_; }

const UnicharAmbigs& tesseract::Dict::getUnicharAmbigs ( )

inline

Definition at line 106 of file dict.h.

                                          {
    return getImage()->getCCUtil()->unichar_ambigs;
  }

const UNICHARSET& tesseract::Dict::getUnicharset ( ) const

inline

Definition at line 100 of file dict.h.

                                          {
    return getImage()->getCCUtil()->unicharset;
  }

UNICHARSET& tesseract::Dict::getUnicharset ( )

inline

Definition at line 103 of file dict.h.

                              {
    return getImage()->getCCUtil()->unicharset;
  }

void tesseract::Dict::go_deeper_dawg_fxn	(	const char *	debug,
		const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		int	char_choice_index,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		bool	word_ending,
		WERD_CHOICE *	word,
		float	certainties[],
		float *	limit,
		WERD_CHOICE *	best_choice,
		int *	attempts_left,
		void *	void_more_args
	)

If the choice being composed so far could be a dictionary word and we have not reached the end of the word keep exploring the char_choices further. Also: – sets hyphen word if needed – if word_ending is true and the word is better than best_choice, copies word to best_choice and logs new word choice

Definition at line 67 of file permdawg.cpp.

                                                                        {
  DawgArgs *more_args = reinterpret_cast<DawgArgs*>(void_more_args);
  word_ending = (char_choice_index == more_args->end_char_choice_index);
  int word_index = word->length() - 1;

  if (ambigs_mode(*limit)) {
    if (best_choice->rating() < *limit) return;
  } else {
    // Prune bad subwords
    if (more_args->rating_array[word_index] == NO_RATING) {
      more_args->rating_array[word_index] = word->rating();
    } else {
      float permdawg_limit = more_args->rating_array[word_index] *
        more_args->rating_margin + kPermDawgRatingPad;
      if (permdawg_limit < word->rating()) {
        if (permute_debug && dawg_debug_level) {
          tprintf("early pruned word rating=%4.2f,"
                  " permdawg_limit=%4.2f, word=%s\n", word->rating(),
                  permdawg_limit, word->debug_string().string());
        }
        return;
      }
    }
  }
  // Deal with hyphens
  if (word_ending && more_args->sought_word_length == kAnyWordLength &&
      has_hyphen_end(*word) && !ambigs_mode(*limit)) {
    // Copy more_args->active_dawgs to clean_active_dawgs removing
    // dawgs of type DAWG_TYPE_PATTERN.
    DawgInfoVector clean_active_dawgs;
    const DawgInfoVector &active_dawgs = *(more_args->active_dawgs);
    for (int i = 0; i < active_dawgs.size(); ++i) {
      if (dawgs_[active_dawgs[i].dawg_index]->type() != DAWG_TYPE_PATTERN) {
        clean_active_dawgs += active_dawgs[i];
      }
    }
    if (clean_active_dawgs.size() > 0) {
      if (permute_debug && dawg_debug_level)
        tprintf("new hyphen choice = %s\n", word->debug_string().string());
      word->set_permuter(more_args->permuter);
      adjust_word(word, certainties, &char_choices, permute_debug);
      set_hyphen_word(*word, *(more_args->active_dawgs),
                      *(more_args->constraints));
      update_best_choice(*word, best_choice);
    }
  } else {  // Look up char in DAWG
    // TODO(daria): update the rest of the code that specifies alternative
    // letter_is_okay_ functions (e.g. TessCharNgram class) to work with
    // multi-byte unichars and/or unichar ids.

    // If the current unichar is an ngram first try calling
    // letter_is_okay() for each unigram it contains separately.
    UNICHAR_ID orig_uch_id = word->unichar_id(word_index);
    bool checked_unigrams = false;
    if (getUnicharset().get_isngram(orig_uch_id)) {
      if (permute_debug && dawg_debug_level) {
        tprintf("checking unigrams in an ngram %s\n",
                getUnicharset().debug_str(orig_uch_id).string());
      }
      int orig_num_fragments = word->fragment_length(word_index);
      int num_unigrams = 0;
      word->remove_last_unichar_id();
      const char *ngram_str = getUnicharset().id_to_unichar(orig_uch_id);
      const char *ngram_str_end = ngram_str + strlen(ngram_str);
      const char *ngram_ptr = ngram_str;
      bool unigrams_ok = true;
      // Construct DawgArgs that reflect the current state.
      DawgInfoVector unigram_active_dawgs = *(more_args->active_dawgs);
      DawgInfoVector unigram_constraints = *(more_args->constraints);
      DawgInfoVector unigram_updated_active_dawgs;
      DawgInfoVector unigram_updated_constraints;
      DawgArgs unigram_dawg_args(&unigram_active_dawgs,
                                 &unigram_constraints,
                                 &unigram_updated_active_dawgs,
                                 &unigram_updated_constraints, 0.0,
                                 more_args->permuter,
                                 more_args->sought_word_length,
                                 more_args->end_char_choice_index);
      // Check unigrams in the ngram with letter_is_okay().
      while (unigrams_ok && ngram_ptr < ngram_str_end) {
        int step = getUnicharset().step(ngram_ptr);
        UNICHAR_ID uch_id = (step <= 0) ? INVALID_UNICHAR_ID :
            getUnicharset().unichar_to_id(ngram_ptr, step);
        ngram_ptr += step;
        ++num_unigrams;
        word->append_unichar_id(uch_id, 1, 0.0, 0.0);
        unigrams_ok = unigrams_ok && (this->*letter_is_okay_)(
            &unigram_dawg_args,
            word->unichar_id(word_index+num_unigrams-1),
            word_ending && (ngram_ptr == ngram_str_end));
        (*unigram_dawg_args.active_dawgs) =
          *(unigram_dawg_args.updated_active_dawgs);
        (*unigram_dawg_args.constraints) =
          *(unigram_dawg_args.updated_constraints);
        if (permute_debug && dawg_debug_level) {
          tprintf("unigram %s is %s\n",
                  getUnicharset().debug_str(uch_id).string(),
                  unigrams_ok ? "OK" : "not OK");
        }
      }
      // Restore the word and copy the updated dawg state if needed.
      while (num_unigrams-- > 0) word->remove_last_unichar_id();
      word->append_unichar_id_space_allocated(
          orig_uch_id, orig_num_fragments, 0.0, 0.0);
      if (unigrams_ok) {
        checked_unigrams = true;
        more_args->permuter = unigram_dawg_args.permuter;
        *(more_args->updated_active_dawgs) =
          *(unigram_dawg_args.updated_active_dawgs);
        *(more_args->updated_constraints) =
          *(unigram_dawg_args.updated_constraints);
      }
    }

    // Check which dawgs from the dawgs_ vector contain the word
    // up to and including the current unichar.
    if (checked_unigrams || (this->*letter_is_okay_)(
        more_args, word->unichar_id(word_index), word_ending)) {
      // Add a new word choice
      if (word_ending) {
        if (permute_debug && dawg_debug_level) {
          tprintf("found word = %s\n", word->debug_string().string());
        }
        if (ambigs_mode(*limit) &&
            strcmp(output_ambig_words_file.string(), "") != 0) {
          if (output_ambig_words_file_ == NULL) {
            output_ambig_words_file_ =
                fopen(output_ambig_words_file.string(), "wb+");
            if (output_ambig_words_file_ == NULL) {
              tprintf("Failed to open output_ambig_words_file %s\n",
                      output_ambig_words_file.string());
              exit(1);
            }
          }
          STRING word_str;
          word->string_and_lengths(&word_str, NULL);
          word_str += " ";
          fprintf(output_ambig_words_file_, word_str.string());
        }
        WERD_CHOICE *adjusted_word = word;
        WERD_CHOICE hyphen_tail_word(&getUnicharset());
        if (hyphen_base_size() > 0) {
          hyphen_tail_word = *word;
          remove_hyphen_head(&hyphen_tail_word);
          adjusted_word = &hyphen_tail_word;
        }
        adjusted_word->set_permuter(more_args->permuter);
        if (!ambigs_mode(*limit)) {
          adjust_word(adjusted_word, &certainties[hyphen_base_size()],
                      &char_choices, permute_debug);
        }
        update_best_choice(*adjusted_word, best_choice);
      } else {  // search the next letter
        // Make updated_* point to the next entries in the DawgInfoVector
        // arrays (that were originally created in dawg_permute_and_select)
        ++(more_args->updated_active_dawgs);
        ++(more_args->updated_constraints);
        // Make active_dawgs and constraints point to the updated ones.
        ++(more_args->active_dawgs);
        ++(more_args->constraints);
        permute_choices(debug, char_choices, char_choice_index + 1,
                        prev_char_frag_info, word, certainties, limit,
                        best_choice, attempts_left, more_args);
        // Restore previous state to explore another letter in this position.
        --(more_args->updated_active_dawgs);
        --(more_args->updated_constraints);
        --(more_args->active_dawgs);
        --(more_args->constraints);
      }
    } else {
      if (permute_debug && dawg_debug_level) {
        tprintf("last unichar not OK at index %d in %s\n",
                word_index, word->debug_string().string());
      }
    }
  }
}

void tesseract::Dict::go_deeper_top_fragments_fxn	(	const char *	debug,
		const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		int	char_choice_index,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		bool	word_ending,
		WERD_CHOICE *	word,
		float	certainties[],
		float *	limit,
		WERD_CHOICE *	best_choice,
		int *	attempts_left,
		void *	more_args
	)

While the choice being composed so far could be better than best_choice keeps exploring char_choices. If the end of the word is reached and the word is better than best_choice, copies word to best_choice and logs the new word choice.

go_deeper_top_fragments_fxn

While the choice being composed so far could be better than best_choice keeps exploring char_choices. If the end of the word is reached and the word is better than best_choice, copies word to best_choice and logs the new word choice.

Definition at line 1538 of file permute.cpp.

                                                                   {
  if (word->rating() < *limit) {
    if (word_ending) {
      if (fragments_debug > 1) {
        tprintf("fragments_debug new choice = %s\n",
                word->debug_string().string());
      }
      *limit = word->rating();
      adjust_non_word(word, certainties, &char_choices, permute_debug);
      update_best_choice(*word, best_choice);
    } else {  // search the next letter
      permute_choices(debug, char_choices, char_choice_index + 1,
                      prev_char_frag_info, word, certainties, limit,
                      best_choice, attempts_left, more_args);
    }
  } else {
    if (fragments_debug > 1) {
      tprintf("fragments_debug pruned word (%s, rating=%4.2f, limit=%4.2f)\n",
              word->debug_string().string(), word->rating(), *limit);
    }
  }
}

int tesseract::Dict::good_choice

(

const WERD_CHOICE &

choice

)

Returns true if a good answer is found for the unknown blob rating.

bool tesseract::Dict::has_hyphen_end ( UNICHAR_ID  unichar_id, bool  first_pos  ) const

inline

Check whether the word has a hyphen at the end.

Definition at line 144 of file dict.h.

                                                                          {
    return (last_word_on_line_ && !first_pos &&
            unichar_id == hyphen_unichar_id_);
  }

bool tesseract::Dict::has_hyphen_end ( const WERD_CHOICE &  word ) const

inline

Same as above, but check the unichar at the end of the word.

Definition at line 149 of file dict.h.

                                                            {
    int word_index = word.length() - 1;
    return has_hyphen_end(word.unichar_id(word_index), word_index == 0);
  }

int tesseract::Dict::hyphen_base_size ( ) const

inline

Size of the base word (the part on the line before) of a hyphenated word.

Definition at line 122 of file dict.h.

                                      {
    return this->hyphenated() ? hyphen_word_->length() : 0;
  }

bool tesseract::Dict::hyphenated ( ) const

inline

Returns true if we've recorded the beginning of a hyphenated word.

Definition at line 118 of file dict.h.

                                 { return
    !last_word_on_line_ && hyphen_word_ && GetMaxFixedLengthDawgIndex() < 0;
  }

void tesseract::Dict::incorporate_segcost

(

WERD_CHOICE *

word

)

Incoporate segmentation cost into word rating.

Incorporate segmentation cost into the word rating. This is done through a multiplier wordseg_rating_adjust_factor_ which is determined in bestfirst.cpp during state evaluation. This is not the cleanest way to do this. It would be better to reorganize the SEARCH_STATE to keep track of associated states, or do the rating adjustment outside the permuter in evalaute_state.

Definition at line 409 of file permute.cpp.

                                                {
  if (!word || wordseg_rating_adjust_factor_ <= 0) return;

  float old_rating = word->rating();
  float new_rating = old_rating * wordseg_rating_adjust_factor_;
  word->set_rating(new_rating);
  if (permute_debug)
    tprintf("Permute segadjust %f * %f --> %f\n",
            old_rating, wordseg_rating_adjust_factor_, new_rating);
}

void tesseract::Dict::init_active_dawgs	(	int	sought_word_length,
		DawgInfoVector *	active_dawgs,
		bool	ambigs_mode
	)		const

Fill the given active_dawgs vector with dawgs that could contain the beginning of the word. If hyphenated() returns true, copy the entries from hyphen_active_dawgs_ instead.

Definition at line 643 of file dict.cpp.

                                                     {
  int i;
  if (sought_word_length != kAnyWordLength) {
    // Only search one fixed word length dawg.
    if (sought_word_length <= max_fixed_length_dawgs_wdlen_ &&
        dawgs_[sought_word_length] != NULL) {
      *active_dawgs += DawgInfo(sought_word_length, NO_EDGE);
    }
  } else if (hyphenated()) {
    *active_dawgs = hyphen_active_dawgs_;
    if (dawg_debug_level >= 3) {
      for (i = 0; i < hyphen_active_dawgs_.size(); ++i) {
        tprintf("Adding hyphen beginning dawg [%d, " REFFORMAT "]\n",
                hyphen_active_dawgs_[i].dawg_index,
                hyphen_active_dawgs_[i].ref);
      }
    }
  } else {
    for (i = 0; i < dawgs_.length(); ++i) {
      if (dawgs_[i] != NULL && kBeginningDawgsType[(dawgs_[i])->type()] &&
          !(ambigs_mode && (dawgs_[i])->type() == DAWG_TYPE_PATTERN)) {
        *active_dawgs += DawgInfo(i, NO_EDGE);
        if (dawg_debug_level >= 3) {
          tprintf("Adding beginning dawg [%d, " REFFORMAT "]\n", i, NO_EDGE);
        }
      }
    }
  }
}

void tesseract::Dict::init_constraints

(

DawgInfoVector *

constraints

)

const

If hyphenated() returns true, copy the entries from hyphen_constraints_ into the given constraints vector.

Definition at line 677 of file dict.cpp.

                                                             {
  if (hyphenated()) {
    *constraints = hyphen_constraints_;
    if (dawg_debug_level >= 3) {
      for (int i = 0; i < hyphen_constraints_.size(); ++i) {
        tprintf("Adding hyphen constraint [%d, " REFFORMAT "]\n",
                hyphen_constraints_[i].dawg_index,
                hyphen_constraints_[i].ref);
      }
    }
  }
}

void tesseract::Dict::InitChoiceAccum

(

)

Initializes the data structures used to keep track the good word choices found for a word.

Definition at line 435 of file stopper.cpp.

                           {
  BLOB_WIDTH *BlobWidth, *End;

  if (best_raw_choice_)
    delete best_raw_choice_;
  best_raw_choice_ = NULL;

  if (best_choices_)
    destroy_nodes(best_choices_, DeleteViableChoiceStruct);
  best_choices_ = NIL_LIST;

  if (raw_choices_)
    destroy_nodes(raw_choices_, DeleteViableChoiceStruct);
  raw_choices_ = NIL_LIST;

  EnableChoiceAccum();

  for (BlobWidth = current_segmentation_,
    End = current_segmentation_ + MAX_NUM_CHUNKS;
    BlobWidth < End; *BlobWidth++ = 1);

}

int tesseract::Dict::LengthOfShortestAlphaRun

(

const WERD_CHOICE &

WordChoice

)

Returns the length of the shortest alpha run in WordChoice.

Definition at line 883 of file stopper.cpp.

                                                                {
  int shortest = MAX_INT32;
  int curr_len = 0;
  for (int w = 0; w < WordChoice.length(); ++w) {
    if (getUnicharset().get_isalpha(WordChoice.unichar_id(w))) {
      curr_len++;
    } else if (curr_len > 0) {
      if (curr_len < shortest) shortest = curr_len;
      curr_len = 0;
    }
  }
  if (curr_len > 0 && curr_len < shortest) {
    shortest = curr_len;
  } else if (shortest == MAX_INT32) {
    shortest = 0;
  }
  return shortest;
}

int tesseract::Dict::LetterIsOkay ( void *  void_dawg_args, UNICHAR_ID  unichar_id, bool  word_end  ) const

inline

Calls letter_is_okay_ member function.

Definition at line 563 of file dict.h.

                                                               {
    return (this->*letter_is_okay_)(void_dawg_args, unichar_id, word_end);
  }

void tesseract::Dict::Load

(

)

Initialize Dict class - load dawgs from [lang].traineddata and user-specified wordlist and parttern list.

Definition at line 219 of file dict.cpp.

                {
  STRING name;
  STRING &lang = getImage()->getCCUtil()->lang;

  if (dawgs_.length() != 0) this->End();

  hyphen_unichar_id_ = getUnicharset().unichar_to_id(kHyphenSymbol);

  LoadEquivalenceList(kHyphenLikeUTF8);
  LoadEquivalenceList(kApostropheLikeUTF8);

  TessdataManager &tessdata_manager =
    getImage()->getCCUtil()->tessdata_manager;

  // Load dawgs_.
  if (load_punc_dawg && tessdata_manager.SeekToStart(TESSDATA_PUNC_DAWG)) {
    punc_dawg_ = new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                                  DAWG_TYPE_PUNCTUATION, lang, PUNC_PERM,
                                  dawg_debug_level);
    dawgs_ += punc_dawg_;
  }
  if (load_system_dawg && tessdata_manager.SeekToStart(TESSDATA_SYSTEM_DAWG)) {
    dawgs_ += new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                               DAWG_TYPE_WORD, lang, SYSTEM_DAWG_PERM,
                               dawg_debug_level);
  }
  if (load_number_dawg && tessdata_manager.SeekToStart(TESSDATA_NUMBER_DAWG)) {
    dawgs_ +=
      new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                       DAWG_TYPE_NUMBER, lang, NUMBER_PERM, dawg_debug_level);
  }
  if (load_bigram_dawg && tessdata_manager.SeekToStart(TESSDATA_BIGRAM_DAWG)) {
    bigram_dawg_ = new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                                    DAWG_TYPE_WORD, // doesn't actually matter.
                                    lang,
                                    COMPOUND_PERM,  // doesn't actually matter.
                                    dawg_debug_level);
  }
  if (load_freq_dawg && tessdata_manager.SeekToStart(TESSDATA_FREQ_DAWG)) {
    freq_dawg_ = new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                                  DAWG_TYPE_WORD, lang, FREQ_DAWG_PERM,
                                  dawg_debug_level);
    dawgs_ += freq_dawg_;
  }
  if (load_unambig_dawg &&
      tessdata_manager.SeekToStart(TESSDATA_UNAMBIG_DAWG)) {
    unambig_dawg_ = new SquishedDawg(tessdata_manager.GetDataFilePtr(),
                                     DAWG_TYPE_WORD, lang, SYSTEM_DAWG_PERM,
                                     dawg_debug_level);
    dawgs_ += unambig_dawg_;
  }

  if (((STRING &)user_words_suffix).length() > 0) {
    Trie *trie_ptr = new Trie(DAWG_TYPE_WORD, lang, USER_DAWG_PERM,
                              kMaxUserDawgEdges, getUnicharset().size(),
                              dawg_debug_level);
    name = getImage()->getCCUtil()->language_data_path_prefix;
    name += user_words_suffix;
    if (!trie_ptr->read_word_list(name.string(), getUnicharset(),
                                  Trie::RRP_REVERSE_IF_HAS_RTL)) {
      tprintf("Error: failed to load %s\n", name.string());
      exit(1);
    }
    dawgs_ += trie_ptr;
  }

  if (((STRING &)user_patterns_suffix).length() > 0) {
    Trie *trie_ptr = new Trie(DAWG_TYPE_PATTERN, lang, USER_PATTERN_PERM,
                              kMaxUserDawgEdges, getUnicharset().size(),
                              dawg_debug_level);
    trie_ptr->initialize_patterns(&(getUnicharset()));
    name = getImage()->getCCUtil()->language_data_path_prefix;
    name += user_patterns_suffix;
    if (!trie_ptr->read_pattern_list(name.string(), getUnicharset())) {
      tprintf("Error: failed to load %s\n", name.string());
      exit(1);
    }
    dawgs_ += trie_ptr;
  }

  document_words_ = new Trie(DAWG_TYPE_WORD, lang, DOC_DAWG_PERM,
                             kMaxDocDawgEdges, getUnicharset().size(),
                             dawg_debug_level);
  dawgs_ += document_words_;

  // This dawg is temporary and should not be searched by letter_is_ok.
  pending_words_ = new Trie(DAWG_TYPE_WORD, lang, NO_PERM,
                            kMaxDocDawgEdges, getUnicharset().size(),
                            dawg_debug_level);

  // Load fixed length dawgs if necessary (used for phrase search
  // for non-space delimited languages).
  if (load_fixed_length_dawgs &&
      tessdata_manager.SeekToStart(TESSDATA_FIXED_LENGTH_DAWGS)) {
    ReadFixedLengthDawgs(DAWG_TYPE_WORD, lang, SYSTEM_DAWG_PERM,
                         dawg_debug_level, tessdata_manager.GetDataFilePtr(),
                         &dawgs_, &max_fixed_length_dawgs_wdlen_);
  }

  // Construct a list of corresponding successors for each dawg. Each entry i
  // in the successors_ vector is a vector of integers that represent the
  // indices into the dawgs_ vector of the successors for dawg i.
  successors_.reserve(dawgs_.length());
  for (int i = 0; i < dawgs_.length(); ++i) {
    const Dawg *dawg = dawgs_[i];
    SuccessorList *lst = new SuccessorList();
    for (int j = 0; j < dawgs_.length(); ++j) {
      const Dawg *other = dawgs_[j];
      if (dawg != NULL && other != NULL &&
          (dawg->lang() == other->lang()) &&
          kDawgSuccessors[dawg->type()][other->type()]) *lst += j;
    }
    successors_ += lst;
  }
}

void tesseract::Dict::LoadEquivalenceList

(

const char *

unichar_strings[]

)

Definition at line 354 of file dict.cpp.

                                                            {
  equivalent_symbols_.push_back(GenericVectorEqEq<UNICHAR_ID>());
  const UNICHARSET &unicharset = getUnicharset();
  GenericVectorEqEq<UNICHAR_ID> *equiv_list = &equivalent_symbols_.back();
  for (int i = 0; unichar_strings[i] != 0; i++) {
    UNICHAR_ID unichar_id = unicharset.unichar_to_id(unichar_strings[i]);
    if (unichar_id != INVALID_UNICHAR_ID) {
      equiv_list->push_back(unichar_id);
    }
  }
}

void tesseract::Dict::LogNewChoice	(	FLOAT32	AdjustFactor,
		const float	Certainties[],
		bool	raw_choice,
		WERD_CHOICE *	WordChoice,
		const BLOB_CHOICE_LIST_VECTOR &	blob_choices
	)

Adds Choice to ChoicesList if the adjusted certainty for Choice is within a reasonable range of the best choice in ChoicesList. The ChoicesList list is kept in sorted order by rating. Duplicates are removed. WordChoice is the new choice for current word. AdjustFactor is an adjustment factor which was applied to choice. Certainties are certainties for each char in new choice. raw_choice indicates whether WordChoice is a raw or best choice.

Definition at line 484 of file stopper.cpp.

                                                                     {
  LIST ChoicesList;
  LIST Choices;
  FLOAT32 Threshold;

  if (!keep_word_choices_)
    return;

  if (raw_choice) {
    if (!best_raw_choice_) {
      best_raw_choice_ =
          NewViableChoice(*WordChoice, AdjustFactor, Certainties);
    } else if (WordChoice->rating() < best_raw_choice_->Rating) {
      if (ChoiceSameAs(*WordChoice, best_raw_choice_)) {
        FillViableChoice(*WordChoice, AdjustFactor, Certainties,
                         best_raw_choice_);
      } else {
        delete best_raw_choice_;
        best_raw_choice_ =
          NewViableChoice(*WordChoice, AdjustFactor, Certainties);
      }
    }
    if (!save_raw_choices) return;
    ChoicesList = raw_choices_;
  } else {
    ChoicesList = best_choices_;
  }

  // Throw out obviously bad choices to save some work.
  if (ChoicesList != NIL_LIST) {
    Threshold = StopperAmbigThreshold(BestFactor(ChoicesList), AdjustFactor);
    if (Threshold > -stopper_ambiguity_threshold_offset)
      Threshold = -stopper_ambiguity_threshold_offset;
    if (WordChoice->certainty() - BestCertainty (ChoicesList) < Threshold) {
      // Set the rating of the word to be terrible, so that it does not
      // get chosen as the best choice.
      if (stopper_debug_level >= 2) {
        STRING bad_string;
        WordChoice->string_and_lengths(&bad_string, NULL);
        tprintf("Discarding choice \"%s\" with an overly low certainty"
                " %.4f vs best choice certainty %.4f (Threshold: %.4f)\n",
                bad_string.string(), WordChoice->certainty(),
                BestCertainty(ChoicesList),
                Threshold + BestCertainty(ChoicesList));
      }
      WordChoice->set_rating(WERD_CHOICE::kBadRating);
      return;
    }
  }

  // See if a choice with the same text string has already been found.
  VIABLE_CHOICE NewChoice = NULL;
  Choices = ChoicesList;

  iterate(Choices) {
    if (ChoiceSameAs (*WordChoice, (VIABLE_CHOICE) first_node (Choices))) {
      if (WordChoice->rating() < BestRating (Choices)) {
        NewChoice = (VIABLE_CHOICE) first_node (Choices);
      } else {
        return;
      }
    }
  }

  if (NewChoice) {
    FillViableChoice(*WordChoice, AdjustFactor, Certainties, NewChoice);
    ChoicesList = delete_d(ChoicesList, NewChoice, is_same_node);
  } else {
    NewChoice = NewViableChoice(*WordChoice, AdjustFactor, Certainties);
  }

  // Now we know we're gonna save it, so add the expensive copy.
  NewChoice->SetBlobChoices(blob_choices);

  ChoicesList = s_adjoin (ChoicesList, NewChoice, CmpChoiceRatings);
  if (stopper_debug_level >= 2)
    raw_choice ? PrintViableChoice (stderr, "New Raw Choice:  ", NewChoice) :
      PrintViableChoice (stderr, "New Word Choice:  ", NewChoice);
  if (count (ChoicesList) > tessedit_truncate_wordchoice_log) {
    Choices =
      (LIST) nth_cell (ChoicesList, tessedit_truncate_wordchoice_log);
    destroy_nodes(list_rest (Choices), DeleteViableChoiceStruct);
    set_rest(Choices, NIL_LIST);
  }

  // Update raw_choices_/best_choices_ pointer.
  if (raw_choice) {
    raw_choices_ = ChoicesList;
  } else {
    best_choices_ = ChoicesList;
  }
}

void tesseract::Dict::LogNewSegmentation

(

PIECES_STATE

BlobWidth

)

Updates the blob widths in current_segmentation_ to be the same as provided in BlobWidth. BlobWidth[] contains the number of chunks in each blob in the current segmentation.

Definition at line 463 of file stopper.cpp.

                                                    {
  BLOB_WIDTH *Segmentation;
  for (Segmentation = current_segmentation_; *BlobWidth != 0;
    BlobWidth++, Segmentation++)
  *Segmentation = *BlobWidth;
  *Segmentation = 0;
}

void tesseract::Dict::LogNewSplit

(

int

Blob

)

Given Blob (the index of the blob that was split), adds 1 chunk to the specified blob for each choice in best_choices_ and for best_raw_choice_.

Definition at line 471 of file stopper.cpp.

                               {
  LIST Choices;
  if (best_raw_choice_) AddNewChunk(best_raw_choice_, Blob);
  Choices = best_choices_;
  iterate(Choices) {
    AddNewChunk ((VIABLE_CHOICE) first_node (Choices), Blob);
  }
  Choices = raw_choices_;
  iterate(Choices) {
    AddNewChunk ((VIABLE_CHOICE) first_node (Choices), Blob);
  }
}

VIABLE_CHOICE tesseract::Dict::NewViableChoice	(	const WERD_CHOICE &	WordChoice,
		FLOAT32	AdjustFactor,
		const float	Certainties[]
	)

Allocates a new viable choice data structure, copies WordChoice, Certainties, and current_segmentation_ into it, returns a pointer to the newly created VIABLE_CHOICE. WordChoice is a choice to be converted to a viable choice. AdjustFactor is a factor used to adjust ratings for WordChoice. Certainties contain certainty for each character in WordChoice.

Definition at line 902 of file stopper.cpp.

                                                               {
  int Length = WordChoice.length();
  assert (Length <= MAX_NUM_CHUNKS && Length > 0);
  VIABLE_CHOICE NewChoice = new VIABLE_CHOICE_STRUCT(Length);
  FillViableChoice(WordChoice, AdjustFactor, Certainties, NewChoice);
  return NewChoice;
}

double tesseract::Dict::ngram_probability_in_context	(	const char *	lang,
		const char *	context,
		int	context_bytes,
		const char *	character,
		int	character_bytes
	)

bool tesseract::Dict::NoDangerousAmbig	(	WERD_CHOICE *	BestChoice,
		DANGERR *	fixpt,
		bool	fix_replaceable,
		BLOB_CHOICE_LIST_VECTOR *	Choices,
		bool *	modified_blobs
	)

Definition at line 581 of file stopper.cpp.

                                                  {
  if (stopper_debug_level > 2) {
    tprintf("\nRunning NoDangerousAmbig() for %s\n",
            best_choice->debug_string().string());
  }

  // Construct BLOB_CHOICE_LIST_VECTOR with ambiguities
  // for each unichar id in BestChoice.
  BLOB_CHOICE_LIST_VECTOR ambig_blob_choices;
  int i;
  bool modified_best_choice = false;
  bool ambigs_found = false;
  // For each position in best_choice:
  // -- choose AMBIG_SPEC_LIST that corresponds to unichar_id at best_choice[i]
  // -- initialize wrong_ngram with a single unichar_id at best_choice[i]
  // -- look for ambiguities corresponding to wrong_ngram in the list while
  //    adding the following unichar_ids from best_choice to wrong_ngram
  //
  // Repeat the above procedure twice: first time look through
  // ambigs to be replaced and replace all the ambiguities found;
  // second time look through dangerous ambiguities and construct
  // ambig_blob_choices with fake a blob choice for each ambiguity
  // and pass them to dawg_permute_and_select() to search for
  // ambiguous words in the dictionaries.
  //
  // Note that during the execution of the for loop (on the first pass)
  // if replacements are made the length of best_choice might change.
  for (int pass = 0; pass < (fix_replaceable ? 2 : 1); ++pass) {
    bool replace = (fix_replaceable && pass == 0);
    const UnicharAmbigsVector &table = replace ?
      getUnicharAmbigs().replace_ambigs() : getUnicharAmbigs().dang_ambigs();
    if (!replace) {
      // Initialize ambig_blob_choices with lists containing a single
      // unichar id for the correspoding position in best_choice.
      // best_choice consisting from only the original letters will
      // have a rating of 0.0.
      for (i = 0; i < best_choice->length(); ++i) {
        BLOB_CHOICE_LIST *lst = new BLOB_CHOICE_LIST();
        BLOB_CHOICE_IT lst_it(lst);
        // TODO(rays/antonova) Should these BLOB_CHOICEs use real xheights
        // or are these fake ones good enough?
        lst_it.add_to_end(new BLOB_CHOICE(best_choice->unichar_id(i),
                                          0.0, 0.0, -1, -1, -1, 0, 1, false));
        ambig_blob_choices.push_back(lst);
      }
    }
    UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE + 1];
    int wrong_ngram_index;
    int next_index;
    int blob_index = 0;
    for (i = 0; i < best_choice->length(); ++i) {
      if (i > 0) blob_index += best_choice->fragment_length(i-1);
      UNICHAR_ID curr_unichar_id = best_choice->unichar_id(i);
      if (stopper_debug_level > 2) {
        tprintf("Looking for %s ngrams starting with %s:\n",
                replace ? "replaceable" : "ambiguous",
                getUnicharset().debug_str(curr_unichar_id).string());
      }
      wrong_ngram_index = 0;
      wrong_ngram[wrong_ngram_index] = curr_unichar_id;
      if (curr_unichar_id == INVALID_UNICHAR_ID ||
          curr_unichar_id >= table.size() ||
          table[curr_unichar_id] == NULL) {
        continue;  // there is no ambig spec for this unichar id
      }
      AmbigSpec_IT spec_it(table[curr_unichar_id]);
      for (spec_it.mark_cycle_pt(); !spec_it.cycled_list();) {
        const AmbigSpec *ambig_spec = spec_it.data();
        wrong_ngram[wrong_ngram_index+1] = INVALID_UNICHAR_ID;
        int compare = UnicharIdArrayUtils::compare(wrong_ngram,
                                                   ambig_spec->wrong_ngram);
        if (stopper_debug_level > 2) {
          tprintf("candidate ngram: ");
          UnicharIdArrayUtils::print(wrong_ngram, getUnicharset());
          tprintf("current ngram from spec: ");
          UnicharIdArrayUtils::print(ambig_spec->wrong_ngram, getUnicharset());
          tprintf("comparison result: %d\n", compare);
        }
        if (compare == 0) {
          // Record the place where we found an ambiguity.
          if (fixpt != NULL) {
            fixpt->push_back(DANGERR_INFO(
                blob_index, blob_index+wrong_ngram_index, replace,
                getUnicharset().get_isngram(ambig_spec->correct_ngram_id)));
            if (stopper_debug_level > 1) {
              tprintf("fixpt+=(%d %d %d %d)\n", blob_index,
                      blob_index+wrong_ngram_index, false,
                      getUnicharset().get_isngram(
                          ambig_spec->correct_ngram_id));
            }
          }

          if (replace) {
            if (stopper_debug_level > 2) {
              tprintf("replace ambiguity with: ");
              UnicharIdArrayUtils::print(
                  ambig_spec->correct_fragments, getUnicharset());
            }
            ReplaceAmbig(i, ambig_spec->wrong_ngram_size,
                         ambig_spec->correct_ngram_id,
                         best_choice, blob_choices, modified_blobs);
            modified_best_choice = true;
          } else if (i > 0 || ambig_spec->type != CASE_AMBIG) {
            // We found dang ambig - update ambig_blob_choices.
            if (stopper_debug_level > 2) {
              tprintf("found ambiguity: ");
              UnicharIdArrayUtils::print(
                  ambig_spec->correct_fragments, getUnicharset());
            }
            ambigs_found = true;
            for (int tmp_index = 0; tmp_index <= wrong_ngram_index;
                 ++tmp_index) {
              // Add a blob choice for the corresponding fragment of the
              // ambiguity. These fake blob choices are initialized with
              // negative ratings (which are not possible for real blob
              // choices), so that dawg_permute_and_select() considers any
              // word not consisting of only the original letters a better
              // choice and stops searching for alternatives once such a
              // choice is found.
              BLOB_CHOICE_IT bc_it(ambig_blob_choices[i+tmp_index]);
              bc_it.add_to_end(new BLOB_CHOICE(
                  ambig_spec->correct_fragments[tmp_index], -1.0, 0.0,
                  -1, -1, -1, 0, 1, false));
            }
          }
          spec_it.forward();
        } else if (compare == -1) {
          if (wrong_ngram_index+1 < ambig_spec->wrong_ngram_size &&
              ((next_index = wrong_ngram_index+1+i) < best_choice->length())) {
            // Add the next unichar id to wrong_ngram and keep looking for
            // more ambigs starting with curr_unichar_id in AMBIG_SPEC_LIST.
            wrong_ngram[++wrong_ngram_index] =
              best_choice->unichar_id(next_index);
          } else {
            break;  // no more matching ambigs in this AMBIG_SPEC_LIST
          }
        } else {
          spec_it.forward();
        }
      }  // end searching AmbigSpec_LIST
    }  // end searching best_choice
  }  // end searching replace and dangerous ambigs

  // If any ambiguities were found permute the constructed ambig_blob_choices
  // to see if an alternative dictionary word can be found.
  if (ambigs_found) {
    if (stopper_debug_level > 2) {
      tprintf("\nResulting ambig_blob_choices:\n");
      for (i = 0; i < ambig_blob_choices.length(); ++i) {
        print_ratings_list("", ambig_blob_choices.get(i), getUnicharset());
        tprintf("\n");
      }
    }
    WERD_CHOICE *alt_word = dawg_permute_and_select(ambig_blob_choices, 0.0);
    ambigs_found = (alt_word->rating() < 0.0);
    if (ambigs_found) {
      if (stopper_debug_level >= 1) {
        tprintf ("Stopper: Possible ambiguous word = %s\n",
                 alt_word->debug_string().string());
      }
      if (fixpt != NULL) {
        // Note: Currently character choices combined from fragments can only
        // be generated by NoDangrousAmbigs(). This code should be updated if
        // the capability to produce classifications combined from character
        // fragments is added to other functions.
        int orig_i = 0;
        for (i = 0; i < alt_word->length(); ++i) {
          bool replacement_is_ngram =
              getUnicharset().get_isngram(alt_word->unichar_id(i));
          int end_i = orig_i + alt_word->fragment_length(i) - 1;
          if (alt_word->fragment_length(i) > 1 ||
              (orig_i == end_i && replacement_is_ngram)) {
            fixpt->push_back(DANGERR_INFO(orig_i, end_i, true,
                                          replacement_is_ngram));
             if (stopper_debug_level > 1) {
               tprintf("fixpt->dangerous+=(%d %d %d %d)\n", orig_i, end_i,
                       true, replacement_is_ngram);
             }
          }
          orig_i += alt_word->fragment_length(i);
        }
      }
    }
    delete alt_word;
  }
  if (output_ambig_words_file_ != NULL) {
    fprintf(output_ambig_words_file_, "\n");
  }

  ambig_blob_choices.delete_data_pointers();
  return !ambigs_found;
}

UNICHAR_ID tesseract::Dict::NormalizeUnicharIdForMatch

(

UNICHAR_ID

unichar_id

)

const

Definition at line 368 of file dict.cpp.

                                                                       {
  for (int i = 0; i < equivalent_symbols_.size(); i++) {
    if (equivalent_symbols_[i].contains(unichar_id)) {
      return equivalent_symbols_[i][0];
    }
  }
  return unichar_id;
}

const int tesseract::Dict::NumDawgs ( ) const

inline

Return the number of dawgs in the dawgs_ vector.

Definition at line 603 of file dict.h.

{ return dawgs_.size(); }

WERD_CHOICE * tesseract::Dict::permute_all	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		const WERD_CHOICE *	best_choice,
		WERD_CHOICE *	raw_choice
	)

Definition at line 331 of file permute.cpp.

                                                        {
  WERD_CHOICE *result1 = NULL;
  WERD_CHOICE *result2 = NULL;
  BOOL8 any_alpha;
  float top_choice_rating_limit = best_choice->rating();
  int word_script_id = get_top_word_script(char_choices, getUnicharset());

  PermuterState permuter_state;
  if (getUnicharset().han_sid() != getUnicharset().null_sid() &&
      word_script_id == getUnicharset().han_sid()) {
    permuter_state.Init(char_choices, getUnicharset(), 1.0f, permute_debug);

    result1 = get_top_choice_word(char_choices);

    // Note that we no longer need the returned word from these permuters,
    // except to delete the memory.  The word choice from all permutations
    // is returned by permuter_state.GetpermutedWord() at the end.
    if (permute_fixed_length_dawg) {
      result2 = permute_fixed_length_words(char_choices, &permuter_state);
      delete result2;
    }
    if (permute_chartype_word) {
      result2 = permute_chartype_words(char_choices, &permuter_state);
      delete result2;
    }
    if (permute_script_word) {
      result2 = permute_script_words(char_choices, &permuter_state);
      delete result2;
    }

    float certainties[MAX_PERM_LENGTH];
    float adjust_factor;
    result2 = permuter_state.GetPermutedWord(certainties, &adjust_factor);
    LogNewChoice(adjust_factor, certainties, false, result2, char_choices);
    result1 = get_best_delete_other(result1, result2);

    if (segment_segcost_rating) incorporate_segcost(result1);
  } else {
    result1 = permute_top_choice(char_choices, &top_choice_rating_limit,
                                 raw_choice, &any_alpha);
    if (result1 == NULL)
      return (NULL);
    if (permute_only_top)
      return result1;

    if (permute_chartype_word) {
      permuter_state.Init(char_choices, getUnicharset(),
                          segment_penalty_garbage, permute_debug);
      result2 = permute_chartype_words(char_choices, &permuter_state);
      result1 = get_best_delete_other(result1, result2);
    }

    // Permute character fragments if necessary.
    if (result1 == NULL || result1->fragment_mark()) {
      result2 = top_fragments_permute_and_select(char_choices,
                                                 top_choice_rating_limit);
      result1 = get_best_delete_other(result1, result2);
    }

    result2 = dawg_permute_and_select(char_choices, best_choice->rating());
    result1 = get_best_delete_other(result1, result2);

    result2 = permute_compound_words(char_choices, best_choice->rating());
    result1 = get_best_delete_other(result1, result2);
  }
  return result1;
}

bool tesseract::Dict::permute_characters	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		WERD_CHOICE *	best_choice,
		WERD_CHOICE *	raw_choice
	)

permute_characters

Permute these characters together according to each of the different permuters that are enabled. Returns true if best_choice was updated.

Definition at line 765 of file permute.cpp.

                                                       {
  if (permute_debug) {
    tprintf("\n\n\n##### Permute_Characters #######\n");
    print_char_choices_list("\n==> Input CharChoices", char_choices,
                            getUnicharset(), segment_debug > 1);
    tprintf("\n");
  }

  if (char_choices.length() == 1 &&
      get_top_choice_uid(char_choices.get(0)) == 0) return false;
  WERD_CHOICE *this_choice = permute_all(char_choices, best_choice, raw_choice);

  if (this_choice && this_choice->rating() < best_choice->rating()) {
    *best_choice = *this_choice;

    if (permute_debug) {
      best_choice->print("\n**** Populate BestChoice");
      cprintf("populate best_choice\n\t%s\n",
              best_choice->debug_string().string());
    }
    delete this_choice;
    return true;
  }
  delete this_choice;
  return false;
}

WERD_CHOICE * tesseract::Dict::permute_chartype_words	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		PermuterState *	permuter_state
	)

checks for consistency in character property (eg. alpah, digit, punct)

Definition at line 542 of file permute.cpp.

                                   {

  if (char_choices.length() >= MAX_PERM_LENGTH)
    return NULL;
  // Store main character property of top choice at every position
  char pos_chartypes[MAX_PERM_LENGTH];
  char word_type = top_word_chartype(char_choices, pos_chartypes);
  if (word_type == 0 || word_type == 'p')
    return NULL;     // skip if word type is punctuation or unknown
  if (permute_debug) {
    tprintf("\n\nPermuteCharType[%c]\n", word_type);
    print_char_choices_list("", char_choices, getUnicharset(), true);
  }

  WERD_CHOICE *current_word = new WERD_CHOICE(&getUnicharset());
  BLOB_CHOICE_IT blob_choice_it;
  const UNICHARSET& unicharset = getUnicharset();
  bool replaced = false;        // has any character choice been replaced
  int prev_unambig_type = 0;    // the last chartype of an unambiguous char
  float certainties[MAX_PERM_LENGTH + 1];
  for (int x = 0; x < char_choices.length(); ++x) {
    BLOB_CHOICE_LIST* pos_choice = char_choices.get(x);
    UNICHAR_ID unichar_id = get_top_choice_uid(pos_choice);
    if (unichar_id == 0) {
      delete current_word;
      return NULL;
    }
    blob_choice_it.set_to_list(pos_choice);
    BLOB_CHOICE *first_choice = blob_choice_it.data();
    ASSERT_HOST(first_choice != NULL);

    const UnicharIdVector* ambig_uids =
        getUnicharAmbigs().OneToOneDefiniteAmbigs(unichar_id);
    bool is_ambiguous = (ambig_uids != NULL);
    bool is_punct = unicharset.get_ispunctuation(unichar_id);
    bool is_consistent = is_punct ||
        unicharset.get_chartype(unichar_id) == prev_unambig_type ||
        unicharset.get_chartype(unichar_id) == word_type;
    bool is_fragment = getUnicharset().get_fragment(unichar_id) != NULL;
    if (permute_debug)
      tprintf("char[%d]:%s is_ambig %c   is_punct %c  is_consistent %c\n",
              x, unicharset.id_to_unichar(unichar_id),
              is_ambiguous?'T':'F', is_punct?'T':'F', is_consistent?'T':'F');

    if (is_fragment) {
      // Ignore any fragmented characters by skipping them to next choice
      // (originally first choice).
      first_choice = get_nth_choice(pos_choice, 1);
      ASSERT_HOST(first_choice != NULL);
    } else if (is_ambiguous && !is_consistent) {
      // Check every confusable blob choice where the top choice is inconsistent
      // with the character type of the previous unambiguous character.
      if (permute_debug) {
        tprintf("Checking %s r%g  PrevCharType %c\n",
                unicharset.id_to_unichar(unichar_id),
                first_choice->rating(), prev_unambig_type);
        print_ratings_list("\t", pos_choice, getUnicharset());
      }
      BLOB_CHOICE* c_it = NULL;
      if (c_it == NULL) {
        c_it = find_choice_by_type(pos_choice, word_type, unicharset);
      }

      // Prefer a character choice whose type is the same as the previous
      // unambiguous character and the confusion appears in the ambig list.
      if (c_it == NULL && prev_unambig_type > 0) {
        c_it = find_choice_by_type(pos_choice, prev_unambig_type, unicharset);
        if (c_it &&
            UnicharIdArrayUtils::find_in(*ambig_uids, c_it->unichar_id()) < 0)
          c_it = NULL;
      }

      // Otherwise, perfer a punctuation
      if (c_it == NULL) {
        c_it = find_choice_by_type(pos_choice, 'p', unicharset);
        if (c_it &&
            UnicharIdArrayUtils::find_in(*ambig_uids, c_it->unichar_id()) < 0)
          c_it = NULL;
      }

      // save any preference other than the top choice
      if (c_it != NULL) {
        if (permute_debug) {
          tprintf("Replacing %s r%g ==> %s r%g\n",
                  unicharset.id_to_unichar(unichar_id), first_choice->rating(),
                  unicharset.id_to_unichar(c_it->unichar_id()), c_it->rating());
          tprintf("\n\nPermuteCharType[%c]\n", word_type);
          print_char_choices_list("", char_choices, getUnicharset(), false);
        }
        if (permuter_state)
          permuter_state->AddPreference(x, c_it, segment_reward_chartype);
        first_choice = c_it;
        replaced = true;
      }
    } else if (!is_ambiguous && !is_punct) {
      // keep the last unambiguous character type
      prev_unambig_type = pos_chartypes[x];
    }
    current_word->append_unichar_id(first_choice->unichar_id(), 1,
                                    first_choice->rating(),
                                    first_choice->certainty());
    certainties[x] = first_choice->certainty();
  }
  // All permuter choices should go through adjust_non_word so the choice
  // rating would be adjusted on the same scale.
  adjust_non_word(current_word, certainties, &char_choices, permute_debug);
  if (replaced) {
    // Apply a reward multiplier on rating if an chartype permutation is made.
    float rating = current_word->rating();
    current_word->set_rating(rating * segment_reward_chartype);
    if (permute_debug)
      current_word->print("<== permute_chartype_word **");
  }
  return current_word;
}

void tesseract::Dict::permute_choices	(	const char *	debug,
		const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		int	char_choice_index,
		const CHAR_FRAGMENT_INFO *	prev_char_frag_info,
		WERD_CHOICE *	word,
		float	certainties[],
		float *	limit,
		WERD_CHOICE *	best_choice,
		int *	attempts_left,
		void *	more_args
	)

permute_choices

Call append_choices() for each BLOB_CHOICE in BLOB_CHOICE_LIST with the given char_choice_index in char_choices.

Definition at line 1437 of file permute.cpp.

                     {
  if (debug) {
    tprintf("%s permute_choices: char_choice_index=%d"
            " limit=%g rating=%g, certainty=%g word=%s\n",
            debug, char_choice_index, *limit, word->rating(),
            word->certainty(), word->debug_string().string());
  }
  if (char_choice_index < char_choices.length()) {
    BLOB_CHOICE_IT blob_choice_it;
    blob_choice_it.set_to_list(char_choices.get(char_choice_index));
    for (blob_choice_it.mark_cycle_pt(); !blob_choice_it.cycled_list();
         blob_choice_it.forward()) {
      (*attempts_left)--;
      append_choices(debug, char_choices, *(blob_choice_it.data()),
                     char_choice_index, prev_char_frag_info, word,
                     certainties, limit, best_choice, attempts_left, more_args);
      if (*attempts_left <= 0) {
        if (debug) tprintf("permute_choices(): attempts_left is 0\n");
        break;
      }
    }
  }
}

WERD_CHOICE * tesseract::Dict::permute_compound_words	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit
	)

permute_compound_words

Return the top choice for each character as the choice for the word.

Definition at line 799 of file permute.cpp.

                        {
  BLOB_CHOICE *first_choice;
  WERD_CHOICE *best_choice = NULL;
  WERD_CHOICE current_word(&getUnicharset(), MAX_WERD_LENGTH);
  int first_index = 0;
  int x;
  BLOB_CHOICE_IT blob_choice_it;

  if (char_choices.length() > MAX_WERD_LENGTH) {
    WERD_CHOICE *bad_word_choice = new WERD_CHOICE(&getUnicharset());
    bad_word_choice->make_bad();
    return bad_word_choice;
  }

  UNICHAR_ID slash = getUnicharset().unichar_to_id("/");
  UNICHAR_ID dash = getUnicharset().unichar_to_id("-");
  for (x = 0; x < char_choices.length(); ++x) {
    blob_choice_it.set_to_list(char_choices.get(x));
    first_choice = blob_choice_it.data();
    if (first_choice->unichar_id() == slash ||
        first_choice->unichar_id() == dash) {
      if (x > first_index) {
        if (segment_debug)
          cprintf ("Hyphenated word found\n");
        permute_subword(char_choices, rating_limit, first_index,
                        x - 1, &current_word);
        if (current_word.rating() > rating_limit)
          break;
      }
      // Append hyphen/slash separator to current_word.
      current_word.append_unichar_id_space_allocated(
          first_choice->unichar_id(), 1,
          first_choice->rating(), first_choice->certainty());

      first_index = x + 1;  // update first_index
    }
  }

  if (first_index > 0 && first_index < x &&
      current_word.rating() <= rating_limit) {
    permute_subword(char_choices, rating_limit, first_index,
                    x - 1, &current_word);
    best_choice = new WERD_CHOICE(current_word);
    best_choice->set_permuter(COMPOUND_PERM);
  }
  return (best_choice);
}

WERD_CHOICE * tesseract::Dict::permute_fixed_length_words	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		PermuterState *	permuter_state
	)

Find permutations matching a list of fixed-char-length dawgs The bestchoice based on this permuter alone is returned. Alternatively, non-conflicting changes can be combined through permuter_state.

Perform search on fixed-length dictionaries within a word. This is used for non-space delimited languages like CJK when a "word" corresponds to a "phrase" consisted of multiple short words. It iterates over every character position looking for longest matches against a set of fixed-length dawgs. Each dictionary hit is rewarded with a rating bonus. Note: this is very slow as it is performed on every segmentation state.

Definition at line 430 of file permute.cpp.

                                   {
  if (permute_debug)
    print_char_choices_list("\n\nPermute FixedLength Word",
                            char_choices, getUnicharset(), false);
  WERD_CHOICE* best_choice =
      new WERD_CHOICE(&getUnicharset(), char_choices.length());
  const int max_dict_len = max_fixed_length_dawgs_wdlen_;
  const int min_dict_len = 2;
  char posstr[256];
  int match_score = 0;
  int anchor_pos = 0;
  while (anchor_pos < char_choices.length()) {
     // search from longest phrase to shortest, stop when we find a match
     WERD_CHOICE* part_choice = NULL;
     int step = max_dict_len;
     while (step >= min_dict_len) {
       int end_pos = anchor_pos + step - 1;
       if (end_pos < char_choices.length()) {
         part_choice = dawg_permute_and_select(char_choices,
                                               200.0,   // rate limit
                                               step,
                                               anchor_pos);
         if (part_choice->length() == step) {
           if (permute_debug)
             tprintf("match found at pos=%d len=%d\n%s\n", anchor_pos, step,
                     part_choice->unichar_string().string());
           break;
         }
         delete part_choice;
         part_choice = NULL;
       }
       step--;
     }

     if (part_choice && step > 1) {   // found lexicon match
       get_posstr_from_choice(char_choices, part_choice, anchor_pos, posstr);
       float adjust_factor = pow(0.95, 1.0 + step*2.0/char_choices.length());
       if (permuter_state)
         permuter_state->AddPreference(anchor_pos, posstr, adjust_factor);
       match_score += step - 1;   // single chars don't count
       if (permute_debug)
         tprintf("Promote word rating %d-len%d\n%s\n", anchor_pos, step,
                 part_choice->unichar_string().string());
     } else {     // no lexicon match
       step = 1;
       part_choice = get_choice_from_posstr(&getUnicharset(), char_choices,
                                            anchor_pos, "0", NULL);
       if (permute_debug)
         tprintf("Single char %d %s\n", anchor_pos,
                 part_choice->unichar_string().string());
     }
     if (part_choice && part_choice->length() > 0)
       (*best_choice) += (*part_choice);
     if (part_choice) delete part_choice;
     anchor_pos += step;
  }

  if (match_score > 0) {
    float adjust_factor = pow(0.8,    // 1.0/segment_penalty_dict_nonword,
                              match_score * 2.0 / char_choices.length());
    float adjusted_score = best_choice->rating() * adjust_factor;
    if (permute_debug)
      tprintf("Adjusting score %f @ %d -> %f\n",
              best_choice->rating(), match_score, adjusted_score);
    best_choice->set_rating(adjusted_score);
  }
  if (permute_debug)
    tprintf("Found Best CJK word %f: %s\n",
            best_choice->rating(), best_choice->unichar_string().string());
  return best_choice;
}

WERD_CHOICE * tesseract::Dict::permute_script_words	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		PermuterState *	permuter_state
	)

Checks for script-consistent permutations. Similar to fixed-length permuter, the best choice is returned by the function, but the combined changes are also recorded into permuter_state.

Try flipping characters in a word to get better script consistency. Similar to how upper/lower case checking is done in top_choice_permuter, this permuter tries to suggest a more script-consistent choice AND modifies the rating. So it combines both the case_ok check and adjust_non_word functionality. However, instead of penalizing an inconsistent word with a > 1 multiplier, we reward the script-consistent choice with a < 1 multiplier.

Definition at line 669 of file permute.cpp.

                                   {
  if (char_choices.length() >= MAX_WERD_LENGTH)
    return NULL;

  int word_sid = get_top_word_script(char_choices, getUnicharset());
  if (word_sid == getUnicharset().null_sid())
    return NULL;

  if (permute_debug) {
    tprintf("\n\nPermuteScript %s\n",
            getUnicharset().get_script_from_script_id(word_sid));
    print_char_choices_list("", char_choices, getUnicharset(),
                            permute_debug > 1);
  }

  WERD_CHOICE *current_word = new WERD_CHOICE(&getUnicharset());
  BLOB_CHOICE_IT blob_choice_it;
  bool replaced = false;
  bool prev_is_consistent = false;
  float certainties[MAX_PERM_LENGTH + 1];
  for (int x = 0; x < char_choices.length(); ++x) {
    blob_choice_it.set_to_list(char_choices.get(x));
    BLOB_CHOICE *first_choice = blob_choice_it.data();
    if (!first_choice) {
      delete current_word;
      return NULL;
    }
    UNICHAR_ID unichar_id = first_choice->unichar_id();
    if (unichar_id == 0) {
      delete current_word;
      return NULL;
    }
    bool sid_consistent = (getUnicharset().get_script(unichar_id) == word_sid);
    bool this_is_punct = getUnicharset().get_chartype(unichar_id) == 'p';
    bool is_fragment = getUnicharset().get_fragment(unichar_id) != NULL;

    if (is_fragment) {
      // Ignore any fragmented characters by skipping them to next choice
      // (originally first choice).
      first_choice = get_nth_choice(char_choices.get(x), 1);
      ASSERT_HOST(first_choice != NULL);
    } else if (!sid_consistent && !this_is_punct && prev_is_consistent) {
      // If the previous char is CJK, we prefer a cjk over non-cjk char
      if (permute_debug) {
        tprintf("Checking %s r%g\n", getUnicharset().id_to_unichar(unichar_id),
                                     first_choice->rating());
        print_ratings_list("\t", char_choices.get(x), getUnicharset());
      }
      // prefer a script consistent choice
      BLOB_CHOICE* c_it = find_choice_by_script(char_choices.get(x),
                                                word_sid, 0, 0);
      // otherwise, prefer a punctuation
      if (c_it == NULL)
        c_it = find_choice_by_type(char_choices.get(x), 'p', getUnicharset());

      if (c_it != NULL) {
        if (permute_debug)
          tprintf("Replacing %s r%g ==> %s r%g\n",
                  getUnicharset().id_to_unichar(unichar_id),
                  first_choice->rating(),
                  getUnicharset().id_to_unichar(c_it->unichar_id()),
                  c_it->rating());
        if (permuter_state)
          permuter_state->AddPreference(x, c_it, segment_reward_script);
        first_choice = c_it;
        replaced = true;
      }
    }
    current_word->append_unichar_id(first_choice->unichar_id(), 1,
                                    first_choice->rating(),
                                    first_choice->certainty());
    certainties[x] = first_choice->certainty();
    prev_is_consistent = sid_consistent;
  }
  // All permuter choices should go through adjust_non_word so the choice
  // rating would be adjusted on the same scale.
  adjust_non_word(current_word, certainties, &char_choices, permute_debug);
  if (replaced) {
    // Apply a reward multiplier on rating if an script permutation is made.
    float rating = current_word->rating();
    current_word->set_rating(rating * segment_reward_script);
    if (permute_debug)
      current_word->print("<== permute_script_word **");
  }
  return current_word;
}

void tesseract::Dict::permute_subword	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit,
		int	start,
		int	end,
		WERD_CHOICE *	current_word
	)

permute_subword

Permute a part of a compound word this subword is bounded by hyphens and the start and end of the word. Call the standard word permute function on a set of choices covering only part of the original word. When it is done reclaim the memory that was used in the exercise.

Definition at line 859 of file permute.cpp.

                                                      {
  int x;
  BLOB_CHOICE_LIST_VECTOR subchoices;
  WERD_CHOICE *best_choice = NULL;
  WERD_CHOICE raw_choice(&getUnicharset());
  raw_choice.make_bad();

  DisableChoiceAccum();

  for (x = start; x <= end; x++) {
    if (char_choices.get(x) != NULL) {
      subchoices += char_choices.get(x);
    }
  }

  if (!subchoices.empty()) {
    WERD_CHOICE initial_choice(&getUnicharset());
    initial_choice.make_bad();
    initial_choice.set_rating(rating_limit);

    best_choice = permute_all(subchoices, &initial_choice, &raw_choice);

    if (best_choice && best_choice->length() > 0) {
      *current_word += *best_choice;
    } else {
      current_word->set_rating(MAX_FLOAT32);
    }
  } else {
    current_word->set_rating(MAX_FLOAT32);
  }

  if (best_choice)
    delete best_choice;

  if (segment_debug && current_word->rating() < MAX_FLOAT32) {
    cprintf ("Subword permuted = %s, %5.2f, %5.2f\n\n",
             current_word->debug_string().string(),
             current_word->rating(), current_word->certainty());
  }
  EnableChoiceAccum();
}

WERD_CHOICE * tesseract::Dict::permute_top_choice	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float *	rating_limit,
		WERD_CHOICE *	raw_choice,
		BOOL8 *	any_alpha
	)

permute_top_choice

Return the top choice for each character as the choice for the word. In addition a choice is created for the best lower and upper case non-words. In each character position the best lower (or upper) case character is substituted for the best overall character.

Definition at line 934 of file permute.cpp.

                      {
  BLOB_CHOICE *first_choice;
  const char *first_char;             //first choice
  const char *second_char;            //second choice
  const char *third_char;             //third choice
  char prev_char[UNICHAR_LEN + 1];    //prev in word
  const char *next_char = "";         //next in word
  const char *next_next_char = "";    //after next next in word

  WERD_CHOICE word(&getUnicharset(), MAX_PERM_LENGTH);
  word.set_permuter(TOP_CHOICE_PERM);
  WERD_CHOICE capital_word(&getUnicharset(), MAX_PERM_LENGTH);
  capital_word.set_permuter(UPPER_CASE_PERM);
  WERD_CHOICE lower_word(&getUnicharset(), MAX_PERM_LENGTH);
  lower_word.set_permuter(LOWER_CASE_PERM);

  int x;
  BOOL8 char_alpha;
  float first_rating = 0;

  float certainties[MAX_PERM_LENGTH + 1];
  float lower_certainties[MAX_PERM_LENGTH + 1];
  float upper_certainties[MAX_PERM_LENGTH + 1];

  BLOB_CHOICE_IT blob_choice_it;
  UNICHAR_ID temp_id;
  UNICHAR_ID unichar_id;
  UNICHAR_ID space = getUnicharset().unichar_to_id(" ");
  register const char* ch;
  register inT8 lower_done;
  register inT8 upper_done;

  prev_char[0] = '\0';

  if (any_alpha != NULL)
    *any_alpha = FALSE;

  if (char_choices.length() > MAX_PERM_LENGTH) {
    return (NULL);
  }

  for (x = 0; x < char_choices.length(); ++x) {
    if (x + 1 < char_choices.length()) {
      unichar_id = get_top_choice_uid(char_choices.get(x+1));
      next_char = unichar_id != INVALID_UNICHAR_ID ?
        getUnicharset().id_to_unichar(unichar_id) : "";
    } else {
      next_char = "";
    }

    if (x + 2 < char_choices.length()) {
      unichar_id = get_top_choice_uid(char_choices.get(x+2));
      next_next_char = unichar_id != INVALID_UNICHAR_ID ?
        getUnicharset().id_to_unichar(unichar_id) : "";
    } else {
      next_next_char = "";
    }

    blob_choice_it.set_to_list(char_choices.get(x));
    ASSERT_HOST(!blob_choice_it.empty());
    first_choice = NULL;
    for (blob_choice_it.mark_cycle_pt(); !blob_choice_it.cycled_list();
         blob_choice_it.forward()) {  // find the best non-fragment char choice
      temp_id = blob_choice_it.data()->unichar_id();
      if (!(getUnicharset().get_fragment(temp_id))) {
        first_choice = blob_choice_it.data();
        break;
      } else if (char_choices.length() > 1) {
        word.set_fragment_mark(true);
        capital_word.set_fragment_mark(true);
        lower_word.set_fragment_mark(true);
      }
    }
    if (first_choice == NULL) {
      cprintf("Permuter found only fragments for"
              " character at position %d; word=%s\n",
              x, word.debug_string().string());
    }
    ASSERT_HOST(first_choice != NULL);

    unichar_id = first_choice->unichar_id() != INVALID_UNICHAR_ID ?
      first_choice->unichar_id() : space;
    first_char = getUnicharset().id_to_unichar(unichar_id);
    first_rating = first_choice->rating();
    word.append_unichar_id_space_allocated(
        unichar_id, 1, first_choice->rating(), first_choice->certainty());
    capital_word.append_unichar_id_space_allocated(
        unichar_id, 1, first_choice->rating(), first_choice->certainty());
    lower_word.append_unichar_id_space_allocated(
        unichar_id, 1, first_choice->rating(), first_choice->certainty());

    certainties[x] = first_choice->certainty();
    lower_certainties[x] = first_choice->certainty();
    upper_certainties[x] = first_choice->certainty();

    lower_done = FALSE;
    upper_done = FALSE;
    char_alpha = FALSE;
    second_char = "";
    third_char = "";
    for (; !blob_choice_it.cycled_list(); blob_choice_it.forward()) {
      unichar_id = blob_choice_it.data()->unichar_id();
      if (getUnicharset().eq(unichar_id, "l") && !blob_choice_it.at_last() &&
          blob_choice_it.data_relative(1)->rating() == first_rating) {
        temp_id = blob_choice_it.data_relative(1)->unichar_id();
        if (getUnicharset().eq(temp_id, "1") ||
            getUnicharset().eq(temp_id, "I")) {
          second_char = getUnicharset().id_to_unichar(temp_id);
          blob_choice_it.forward();
          if (!blob_choice_it.at_last() &&
              blob_choice_it.data_relative(1)->rating() == first_rating) {
            temp_id = blob_choice_it.data_relative(1)->unichar_id();
            if (getUnicharset().eq(temp_id, "1") ||
                getUnicharset().eq(temp_id, "I")) {
              third_char = getUnicharset().id_to_unichar(temp_id);
              blob_choice_it.forward();
            }
          }
          ch = choose_il1 (first_char, second_char, third_char,
            prev_char, next_char, next_next_char);
          unichar_id = (ch != NULL && *ch != '\0') ?
            getUnicharset().unichar_to_id(ch) : INVALID_UNICHAR_ID;
          if (strcmp(ch, "l") != 0 &&
              getUnicharset().eq(word.unichar_id(x), "l")) {
            word.set_unichar_id(unichar_id, x);
            lower_word.set_unichar_id(unichar_id, x);
            capital_word.set_unichar_id(unichar_id, x);
          }
        }
      }
      if (unichar_id != INVALID_UNICHAR_ID) {
        /* Find lower case */
        if (!lower_done &&
            (getUnicharset().get_islower(unichar_id) ||
             (getUnicharset().get_isupper(unichar_id) && x == 0))) {
          lower_word.set_unichar_id(unichar_id, x);
          lower_word.set_rating(lower_word.rating() -
            first_choice->rating() + blob_choice_it.data()->rating());
          if (blob_choice_it.data()->certainty() < lower_word.certainty()) {
            lower_word.set_certainty(blob_choice_it.data()->certainty());
          }
          lower_certainties[x] = blob_choice_it.data()->certainty();
          lower_done = TRUE;
        }
        /* Find upper case */
        if (!upper_done && getUnicharset().get_isupper(unichar_id)) {
          capital_word.set_unichar_id(unichar_id, x);
          capital_word.set_rating(capital_word.rating() -
            first_choice->rating() + blob_choice_it.data()->rating());
          if (blob_choice_it.data()->certainty() < capital_word.certainty()) {
            capital_word.set_certainty(blob_choice_it.data()->certainty());
          }
          upper_certainties[x] = blob_choice_it.data()->certainty();
          upper_done = TRUE;
        }
        if (!char_alpha) {
          const CHAR_FRAGMENT *fragment =
            getUnicharset().get_fragment(unichar_id);
          temp_id = !fragment ? unichar_id :
            getUnicharset().unichar_to_id(fragment->get_unichar());
          if (getUnicharset().get_isalpha(temp_id)) {
            char_alpha = TRUE;
          }
        }
        if (lower_done && upper_done)
          break;
      }
    }
    if (char_alpha && any_alpha != NULL)
      *any_alpha = TRUE;

    if (word.rating() > bestrate_pruning_factor * *rating_limit) {
      if (permute_debug)
        tprintf("\n***** Aborting high-cost word: %g > limit %g\n",
                word.rating(), bestrate_pruning_factor * *rating_limit);
      return (NULL);
    }

    *prev_char = '\0';
    temp_id = word.unichar_id(word.length()-1);
    if (temp_id != INVALID_UNICHAR_ID) {
      strcpy(prev_char, getUnicharset().id_to_unichar(temp_id));
    }
  }

  if (raw_choice != NULL && word.rating() < raw_choice->rating()) {
    *raw_choice = word;
    LogNewChoice(1.0, certainties, true, raw_choice, char_choices);
  }
  float rating = word.rating();
  adjust_non_word(&word, certainties, &char_choices, permute_debug);

  float lower_rating = lower_word.rating();
  adjust_non_word(&lower_word, lower_certainties, &char_choices,
                  permute_debug);

  float upper_rating = capital_word.rating();
  adjust_non_word(&capital_word, upper_certainties, &char_choices,
                  permute_debug);

  WERD_CHOICE *best_choice = &word;
  *rating_limit = rating;
  if (lower_word.rating() < best_choice->rating()) {
    best_choice = &lower_word;
    *rating_limit = lower_rating;
  }
  if (capital_word.rating() < best_choice->rating()) {
    best_choice = &capital_word;
    *rating_limit = upper_rating;
  }
  return new WERD_CHOICE(*best_choice);
}

void tesseract::Dict::PrintAmbigAlternatives	(	FILE *	file,
		const char *	label,
		int	label_num_unichars
	)

Print all the choices in raw_choices_ list for non 1-1 ambiguities.

Definition at line 358 of file stopper.cpp.

                                                          {
  iterate(raw_choices_) {
    VIABLE_CHOICE Choice = (VIABLE_CHOICE)first_node(raw_choices_);
    if (Choice->Length > 0 &&
        (label_num_unichars > 1 || Choice->Length > 1)) {
      for (int i = 0; i < Choice->Length; i++) {
        fprintf(file, "%s",
                getUnicharset().id_to_unichar(Choice->Blob[i].Class));
      }
      fflush(file);
      fprintf(file, "\t%s\t%.4f\t%.4f\n", label,
              Choice->Rating, Choice->Certainty);
    }
  }
}

void tesseract::Dict::PrintViableChoice	(	FILE *	File,
		const char *	Label,
		VIABLE_CHOICE	Choice
	)

Dumps a text representation of the specified Choice to File.

Definition at line 912 of file stopper.cpp.

                                                                                {
  int i, j;
  fprintf (File, "%s", Label);
  fprintf(File, "(R=%5.1f, C=%4.1f, F=%4.2f, Frag=%d)  ",
    Choice->Rating, Choice->Certainty,
    Choice->AdjustFactor, Choice->ComposedFromCharFragments);

  for (i = 0; i < Choice->Length; i++)
    fprintf(File, "%s", getUnicharset().id_to_unichar(Choice->Blob[i].Class));
  fprintf(File, "\n");

  for (i = 0; i < Choice->Length; i++) {
    fprintf(File, "  %s", getUnicharset().id_to_unichar(Choice->Blob[i].Class));
    for (j = 0; j < Choice->Blob[i].NumChunks - 1; j++)
      fprintf(File, "    ");
  }
  fprintf(File, "\n");

  for (i = 0; i < Choice->Length; i++) {
    for (j = 0; j < Choice->Blob[i].NumChunks; j++)
      fprintf(File, "%3d ", (int) (Choice->Blob[i].Certainty * -10.0));
  }
  fprintf(File, "\n");

  for (i = 0; i < Choice->Length; i++) {
    for (j = 0; j < Choice->Blob[i].NumChunks; j++)
      fprintf(File, "%3d ", Choice->Blob[i].NumChunks);
  }
  fprintf(File, "\n");
}

double tesseract::Dict::ProbabilityInContext ( const char *  context, int  context_bytes, const char *  character, int  character_bytes  )

inline

Calls probability_in_context_ member function.

Definition at line 576 of file dict.h.

                                                   {
    return (this->*probability_in_context_)(
        getImage()->getCCUtil()->lang.string(),
        context, context_bytes,
        character, character_bytes);
  }

void tesseract::Dict::ProcessPatternEdges	(	const Dawg *	dawg,
		const DawgInfo &	info,
		UNICHAR_ID	unichar_id,
		bool	word_end,
		DawgArgs *	dawg_args,
		PermuterType *	current_permuter
	)		const

For each of the character classes of the given unichar_id (and the unichar_id itself) finds the corresponding outgoing node or self-loop in the given dawg and (after checking that it is valid) records it in dawg_args->updated_ative_dawgs. Updates current_permuter if any valid edges were found.

Definition at line 554 of file dict.cpp.

                                                              {
  NODE_REF node = GetStartingNode(dawg, info.ref);
  // Try to find the edge corresponding to the exact unichar_id and to all the
  // edges corresponding to the character class of unichar_id.
  GenericVector<UNICHAR_ID> unichar_id_patterns;
  unichar_id_patterns.push_back(unichar_id);
  dawg->unichar_id_to_patterns(unichar_id, getUnicharset(),
                               &unichar_id_patterns);
  for (int i = 0; i < unichar_id_patterns.size(); ++i) {
    // On the first iteration check all the outgoing edges.
    // On the second iteration check all self-loops.
    for (int k = 0; k < 2; ++k) {
    EDGE_REF edge = (k == 0) ?
      dawg->edge_char_of(node, unichar_id_patterns[i], word_end)
      : dawg->pattern_loop_edge(info.ref, unichar_id_patterns[i], word_end);
      if (edge != NO_EDGE) {
        if (dawg_debug_level >= 3) {
          tprintf("Pattern dawg: [%d, " REFFORMAT "] edge=" REFFORMAT "\n",
                  info.dawg_index, node, edge);
        }
        if (ConstraintsOk(*(dawg_args->updated_constraints),
                          word_end, dawg->type())) {
          if (dawg_debug_level >=3) {
            tprintf("Letter found in pattern dawg %d\n", info.dawg_index);
          }
          if (dawg->permuter() > *curr_perm) *curr_perm = dawg->permuter();
          dawg_args->updated_active_dawgs->add_unique(
              DawgInfo(info.dawg_index, edge), dawg_debug_level > 0,
              "Append current dawg to updated active dawgs: ");
        }
      }
    }
  }
}

void tesseract::Dict::ReadFixedLengthDawgs ( DawgType  type, const STRING &  lang, PermuterType  perm, int  debug_level, FILE *  file, DawgVector *  dawg_vec, int *  max_wdlen  )

static

Read/Write/Access special purpose dawgs which contain words only of a certain length (used for phrase search for non-space-delimited languages). Reads a sequence of dawgs from the given file. Appends the constructed dawgs to the given dawg_vec. Fills the given table with indices of the dawgs in the dawg_vec corresponding to the dawgs with words of a particular length.

Definition at line 592 of file dict.cpp.

                                                {
  int i;
  DawgVector dawg_vec_copy;
  dawg_vec_copy.move(dawg_vec); // save the input dawg_vec.
  inT32 num_dawgs;
  fread(&num_dawgs, sizeof(inT32), 1, file);
  bool swap = (num_dawgs > MAX_WERD_LENGTH);
  if (swap) num_dawgs = reverse32(num_dawgs);
  inT32 word_length;
  int max_word_length = 0;
  // Read and record pointers to fixed-length dawgs such that:
  // dawg_vec[word_length] = pointer to dawg with word length of word_length,
  //                         NULL if such fixed-length dawg does not exist.
  for (i = 0; i < num_dawgs; ++i) {
    fread(&word_length, sizeof(inT32), 1, file);
    if (swap) word_length = reverse32(word_length);
    ASSERT_HOST(word_length >  0 && word_length <= MAX_WERD_LENGTH);
    while (word_length >= dawg_vec->size()) dawg_vec->push_back(NULL);
    (*dawg_vec)[word_length] =
      new SquishedDawg(file, type, lang, perm, debug_level);
    if (word_length > max_word_length) max_word_length = word_length;
  }
  *max_wdlen = max_word_length;
  // Entries dawg_vec[0] to dawg_vec[max_word_length] now hold pointers
  // to fixed-length dawgs. The rest of the vector will contain the dawg
  // pointers from the original input dawg_vec.
  for (i = 0; i < dawg_vec_copy.size(); ++i) {
    dawg_vec->push_back(dawg_vec_copy[i]);
  }
}

void tesseract::Dict::remove_hyphen_head ( WERD_CHOICE *  word ) const

inline

Erase the unichar ids corresponding to the portion of the word from the previous line. The word is not changed if it is not split between lines and hyphenated.

Definition at line 137 of file dict.h.

                                                          {
    if (this->hyphenated()) {
      word->remove_unichar_ids(0, hyphen_word_->length());
      if (hyphen_debug_level) hyphen_word_->print("remove_hyphen_head: ");
    }
  }

void tesseract::Dict::ReplaceAmbig	(	int	wrong_ngram_begin_index,
		int	wrong_ngram_size,
		UNICHAR_ID	correct_ngram_id,
		WERD_CHOICE *	werd_choice,
		BLOB_CHOICE_LIST_VECTOR *	blob_choices,
		bool *	modified_blobs
	)

Replaces the corresponding wrong ngram in werd_choice with the correct one. We indicate that this newly inserted ngram unichar is composed from several fragments and modify the corresponding entries in blob_choices to contain fragments of the correct ngram unichar instead of the original unichars. Ratings and certainties of entries in blob_choices and werd_choice are unichaged. E.g. for werd_choice mystring'' and ambiguity ''->": werd_choice becomes mystring", first ' in blob_choices becomes |"|0|2, second one is set to |"|1|2.

Definition at line 802 of file stopper.cpp.

                                              {
  int num_blobs_to_replace = 0;
  int begin_blob_index = 0;
  int i;
  for (i = 0; i < wrong_ngram_begin_index + wrong_ngram_size; ++i) {
    if (i >= wrong_ngram_begin_index) {
      num_blobs_to_replace +=  werd_choice->fragment_length(i);
    } else {
      begin_blob_index += werd_choice->fragment_length(i);
    }
  }
  BLOB_CHOICE_IT bit;
  int temp_blob_index = begin_blob_index;
  const char *temp_uch = NULL;
  const char *correct_ngram_str =
    getUnicharset().id_to_unichar(correct_ngram_id);
  for (int replaced_count = 0; replaced_count < wrong_ngram_size;
       ++replaced_count) {
    if (blob_choices != NULL) {
      UNICHAR_ID uch_id = werd_choice->unichar_id(wrong_ngram_begin_index);
      int fraglen = werd_choice->fragment_length(wrong_ngram_begin_index);
      if (fraglen > 1) temp_uch = getUnicharset().id_to_unichar(uch_id);
      for (i = 0; i < fraglen; ++i) {
        if (fraglen > 1) {
          STRING frag_str =
            CHAR_FRAGMENT::to_string(temp_uch, i, fraglen, false);
          getUnicharset().unichar_insert(frag_str.string());
          uch_id = getUnicharset().unichar_to_id(frag_str.string());
        }
        bit.set_to_list(blob_choices->get(temp_blob_index));
        STRING correct_frag_uch =
          CHAR_FRAGMENT::to_string(correct_ngram_str,
                                   temp_blob_index - begin_blob_index,
                                   num_blobs_to_replace, false);
        getUnicharset().unichar_insert(correct_frag_uch.string());
        UNICHAR_ID correct_frag_uch_id =
          getUnicharset().unichar_to_id(correct_frag_uch.string());
        // Find the WERD_CHOICE corresponding to the original unichar in
        // the list of blob choices, add the derived character fragment
        // before it with the same rating and certainty.
        for (bit.mark_cycle_pt(); !bit.cycled_list(); bit.forward()) {
          if (bit.data()->unichar_id() == correct_frag_uch_id) {
            break;  // the unichar we want to insert is already there
          }
          if (bit.data()->unichar_id() == uch_id) {
            bit.add_before_then_move(new BLOB_CHOICE(*(bit.data())));
            bit.data()->set_unichar_id(correct_frag_uch_id);
            if (modified_blobs != NULL) *modified_blobs = true;
            break;
          }
        }
        temp_blob_index++;
      }
    }
    // Remove current unichar from werd_choice. On the last iteration
    // set the correct replacement unichar instead of removing a unichar.
    if (replaced_count + 1 == wrong_ngram_size) {
      werd_choice->set_unichar_id(correct_ngram_id,
          num_blobs_to_replace, 0.0, 0.0, wrong_ngram_begin_index);
    } else {
      werd_choice->remove_unichar_id(wrong_ngram_begin_index);
    }
  }
  if (stopper_debug_level >= 1 && modified_blobs != NULL &&
      *modified_blobs && blob_choices != NULL) {
      werd_choice->print("ReplaceAmbig() ");
      tprintf("Modified blob_choices: ");
      for (int i = 0; i < blob_choices->size(); ++i) {
        print_ratings_list("\n", blob_choices->get(i), getUnicharset());
    }
  }
}

void tesseract::Dict::reset_hyphen_vars

(

bool

last_word_on_line

)

Unless the previous word was the last one on the line, and the current one is not (thus it is the first one on the line), erase hyphen_word_, clear hyphen_active_dawgs_, hyphen_constraints_ update last_word_on_line_.

Definition at line 32 of file hyphen.cpp.

                                                   {
  if (!(last_word_on_line_ == true && last_word_on_line == false)) {
    if (hyphen_word_ != NULL) {
      delete hyphen_word_;
      hyphen_word_ = NULL;
      hyphen_active_dawgs_.clear();
      hyphen_constraints_.clear();
    }
  }
  if (hyphen_debug_level) {
    tprintf("reset_hyphen_vars: last_word_on_line %d -> %d\n",
            last_word_on_line_, last_word_on_line);
  }
  last_word_on_line_ = last_word_on_line;
}

void tesseract::Dict::ResetDocumentDictionary ( )

inline

Definition at line 478 of file dict.h.

                                 {
    if (pending_words_ != NULL)
      pending_words_->clear();
    if (document_words_ != NULL)
      document_words_->clear();
  }

void tesseract::Dict::set_hyphen_word	(	const WERD_CHOICE &	word,
		const DawgInfoVector &	active_dawgs,
		const DawgInfoVector &	constraints
	)

Update hyphen_word_, and copy the given DawgInfoVectors into hyphen_active_dawgs_ and hyphen_constraints_.

Definition at line 50 of file hyphen.cpp.

                                                              {
  if (hyphen_word_ == NULL) {
    hyphen_word_ = new WERD_CHOICE(word.unicharset());
    hyphen_word_->make_bad();
  }
  if (hyphen_word_->rating() > word.rating()) {
    *hyphen_word_ = word;
    // Remove the last unichar id as it is a hyphen, and remove
    // any unichar_string/lengths that are present.
    hyphen_word_->remove_last_unichar_id();
    hyphen_active_dawgs_ = active_dawgs;
    hyphen_constraints_ = constraints;
  }
  if (hyphen_debug_level) {
    hyphen_word_->print("set_hyphen_word: ");
  }
}

void tesseract::Dict::SettupStopperPass1

(

)

Sets up stopper variables in preparation for the first pass.

Definition at line 780 of file stopper.cpp.

                              {
  reject_offset_ = 0.0;
}

void tesseract::Dict::SettupStopperPass2

(

)

Sets up stopper variables in preparation for the second pass.

Definition at line 784 of file stopper.cpp.

                              {
  reject_offset_ = stopper_phase2_certainty_rejection_offset;
}

void tesseract::Dict::SetWordsegRatingAdjustFactor ( float  f )

inline

Set wordseg_rating_adjust_factor_ to the given value.

Definition at line 726 of file dict.h.

                                                    {
    wordseg_rating_adjust_factor_ = f;
  }

double tesseract::Dict::StopperAmbigThreshold ( double  f1, double  f2  )

inline

Definition at line 322 of file dict.h.

                                                     {
    return (f2 - f1) * stopper_ambiguity_threshold_gain -
        stopper_ambiguity_threshold_offset;
  }

bool tesseract::Dict::StringSameAs	(	const WERD_CHOICE &	WordChoice,
		VIABLE_CHOICE	ViableChoice
	)

Compares unichar ids in word_choice to those in viable_choice, returns true if they are the same.

Definition at line 951 of file stopper.cpp.

                                                    {
  if (WordChoice.length() != ViableChoice->Length) {
    return false;
  }
  int i;
  CHAR_CHOICE *CharChoice;
  for (i = 0, CharChoice = &(ViableChoice->Blob[0]);
       i < ViableChoice->Length; CharChoice++, i++) {
    if (CharChoice->Class != WordChoice.unichar_id(i)) {
      return false;
    }
  }
  return true;
}

bool tesseract::Dict::StringSameAs	(	const char *	String,
		const char *	String_lengths,
		VIABLE_CHOICE	ViableChoice
	)

Compares String to ViableChoice and returns true if they are the same.

Definition at line 967 of file stopper.cpp.

                                                    {
  CHAR_CHOICE *Char;
  int i;
  int current_unichar_length;

  for (Char = &(ViableChoice->Blob[0]), i = 0;
    i < ViableChoice->Length;
       String += *(String_lengths++), Char++, i++) {
    current_unichar_length = strlen(getUnicharset().id_to_unichar(Char->Class));
  if (current_unichar_length != *String_lengths ||
      strncmp(String, getUnicharset().id_to_unichar(Char->Class),
              current_unichar_length) != 0)
    return false;
  }
  return (*String == 0) ? true : false;
}

WERD_CHOICE * tesseract::Dict::top_fragments_permute_and_select	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		float	rating_limit
	)

top_fragments_permute_and_select

Creates a copy of character choices list that contain only fragments and the best non-fragmented character choice. Permutes character in this shortened list, builds characters from fragments if possible and returns a better choice if found.

Definition at line 1367 of file permute.cpp.

                        {
  if (char_choices.length() <= 1 ||
      char_choices.length() > MAX_PERM_LENGTH) {
    return NULL;
  }
  // See it would be possible to benefit from permuting fragments.
  int x;
  float min_rating = 0.0;
  BLOB_CHOICE_IT blob_choice_it;
  for (x = 0; x < char_choices.length(); ++x) {
    blob_choice_it.set_to_list(char_choices.get(x));
    if (blob_choice_it.data()) {
      min_rating += blob_choice_it.data()->rating();
    }
    if (min_rating >= rating_limit) {
      return NULL;
    }
  }
  if (fragments_debug > 1) {
    tprintf("A choice with fragment beats top choice\n");
    tprintf("Running fragment permuter...\n");
  }

  // Construct a modified choices list that contains (for each position):
  // the best choice, all fragments and at least one choice for
  // a non-fragmented character.
  BLOB_CHOICE_LIST_VECTOR frag_char_choices(char_choices.length());
  for (x = 0; x < char_choices.length(); ++x) {
    bool need_nonfrag_char = true;
    BLOB_CHOICE_LIST *frag_choices = new BLOB_CHOICE_LIST();
    BLOB_CHOICE_IT frag_choices_it;
    frag_choices_it.set_to_list(frag_choices);
    blob_choice_it.set_to_list(char_choices.get(x));
    for (blob_choice_it.mark_cycle_pt(); !blob_choice_it.cycled_list();
         blob_choice_it.forward()) {
      if (getUnicharset().get_fragment(blob_choice_it.data()->unichar_id())) {
        frag_choices_it.add_after_then_move(
            new BLOB_CHOICE(*(blob_choice_it.data())));
      } else if (need_nonfrag_char) {
        frag_choices_it.add_after_then_move(
            new BLOB_CHOICE(*(blob_choice_it.data())));
        need_nonfrag_char = false;
      }
    }
    frag_char_choices += frag_choices;
  }

  WERD_CHOICE *best_choice = new WERD_CHOICE(&getUnicharset());
  best_choice->make_bad();
  WERD_CHOICE word(&getUnicharset(), MAX_PERM_LENGTH);
  word.set_permuter(TOP_CHOICE_PERM);
  float certainties[MAX_PERM_LENGTH];
  this->go_deeper_fxn_ = &tesseract::Dict::go_deeper_top_fragments_fxn;
  int attempts_left = max_permuter_attempts;
  permute_choices((fragments_debug > 1) ? "fragments_debug" : NULL,
                  frag_char_choices, 0, NULL, &word, certainties,
                  &rating_limit, best_choice, &attempts_left, NULL);

  frag_char_choices.delete_data_pointers();
  return best_choice;
}

char tesseract::Dict::top_word_chartype	(	const BLOB_CHOICE_LIST_VECTOR &	char_choices,
		char *	pos_chartypes
	)

Look up the main chartype for each character position and store it in the given array. Also returns the dominant type from unambiguous top choices.

Definition at line 512 of file permute.cpp.

                                                  {
  const UNICHARSET &unicharset = getUnicharset();
  const int hist_size = 128;   // to contain 'A','a','0','x','p'
  int chprop[hist_size];
  int x;
  for (x = 0; x < hist_size; x++) chprop[x] = 0;
  for (x = 0; x < char_choices.length(); ++x) {
    UNICHAR_ID unichar_id = get_top_choice_uid(char_choices.get(x));
    char ctype = unicharset.get_chartype(unichar_id);
    if (pos_chartypes) pos_chartypes[x] = ctype;
    if (ctype == 0 || ctype == 'p') continue;
    if (getUnicharAmbigs().OneToOneDefiniteAmbigs(unichar_id) != NULL) continue;
    chprop[ctype]++;
    if (x == 0 && ctype == 'A')  // first-cap also counts as lower
      chprop['a']++;
  }
  int max_prop = 0;
  for (x = 1; x < hist_size; x++)
    if (chprop[x] >= chprop[max_prop]) max_prop = x;
  return (chprop[max_prop] > 0) ? max_prop : 0;
}

int tesseract::Dict::UniformCertainties	(	const BLOB_CHOICE_LIST_VECTOR &	Choices,
		const WERD_CHOICE &	BestChoice
	)

Returns true if the certainty of the BestChoice word is within a reasonable range of the average certainties for the best choices for each character in the segmentation. This test is used to catch words in which one character is much worse than the other characters in the word (i.e. false will be returned in that case). The algorithm computes the mean and std deviation of the certainties in the word with the worst certainty thrown out.

Definition at line 986 of file stopper.cpp.

                                                            {
  float Certainty;
  float WorstCertainty = MAX_FLOAT32;
  float CertaintyThreshold;
  FLOAT64 TotalCertainty;
  FLOAT64 TotalCertaintySquared;
  FLOAT64 Variance;
  FLOAT32 Mean, StdDev;
  int WordLength;

  WordLength = Choices.length();
  if (WordLength < 3)
    return true;

  TotalCertainty = TotalCertaintySquared = 0.0;
  BLOB_CHOICE_IT BlobChoiceIt;
  for (int i = 0; i < Choices.length(); ++i) {
    BlobChoiceIt.set_to_list(Choices.get(i));
    Certainty = BlobChoiceIt.data()->certainty();
    TotalCertainty += Certainty;
    TotalCertaintySquared += Certainty * Certainty;
    if (Certainty < WorstCertainty)
      WorstCertainty = Certainty;
  }

  // Subtract off worst certainty from statistics.
  WordLength--;
  TotalCertainty -= WorstCertainty;
  TotalCertaintySquared -= WorstCertainty * WorstCertainty;

  Mean = TotalCertainty / WordLength;
  Variance = ((WordLength * TotalCertaintySquared -
    TotalCertainty * TotalCertainty) /
    (WordLength * (WordLength - 1)));
  if (Variance < 0.0)
    Variance = 0.0;
  StdDev = sqrt (Variance);

  CertaintyThreshold = Mean - stopper_allowable_character_badness * StdDev;
  if (CertaintyThreshold > stopper_nondict_certainty_base)
    CertaintyThreshold = stopper_nondict_certainty_base;

  if (BestChoice.certainty() < CertaintyThreshold) {
    if (stopper_debug_level >= 1)
      cprintf("Stopper: Non-uniform certainty = %4.1f"
              " (m=%4.1f, s=%4.1f, t=%4.1f)\n",
              BestChoice.certainty(), Mean, StdDev, CertaintyThreshold);
    return false;
  } else {
    return true;
  }
}

void tesseract::Dict::update_best_choice ( const WERD_CHOICE &  word, WERD_CHOICE *  best_choice  )

inline

Copies word into best_choice if its rating is smaller than that of best_choice.

Definition at line 166 of file dict.h.

                                                           {
    if (word.rating() < best_choice->rating()) *best_choice = word;
  }

bool tesseract::Dict::valid_bigram	(	const WERD_CHOICE &	word1,
		const WERD_CHOICE &	word2
	)		const

Definition at line 849 of file dict.cpp.

                                                        {
  if (bigram_dawg_ == NULL) return false;

  // Extract the core word from the middle of each word with any digits
  //         replaced with question marks.
  int w1start, w1end, w2start, w2end;
  word1.punct_stripped(&w1start, &w1end);
  word2.punct_stripped(&w2start, &w2end);

  // We don't want to penalize a single guillemet, hyphen, etc.
  // But our bigram list doesn't have any information about punctuation.
  if (w1start >= w1end) return word1.length() < 3;
  if (w2start >= w2end) return word2.length() < 3;

  const UNICHARSET& uchset = getUnicharset();
  STRING bigram_string;
  for (int i = w1start; i < w1end; i++) {
    UNICHAR_ID ch = NormalizeUnicharIdForMatch(word1.unichar_id(i));
    bigram_string += uchset.get_isdigit(ch) ? "?" : uchset.id_to_unichar(ch);
  }
  bigram_string += " ";
  for (int i = w2start; i < w2end; i++) {
    UNICHAR_ID ch = NormalizeUnicharIdForMatch(word2.unichar_id(i));
    bigram_string += uchset.get_isdigit(ch) ? "?" : uchset.id_to_unichar(ch);
  }
  WERD_CHOICE normalized_word(bigram_string.string(), uchset);
  return bigram_dawg_->word_in_dawg(normalized_word);
}

bool tesseract::Dict::valid_punctuation

(

const WERD_CHOICE &

word

)

Returns true if the word contains a valid punctuation pattern. Note: Since the domains of punctuation symbols and symblos used in numbers are not disjoint, a valid number might contain an invalid punctuation pattern (e.g. .99).

Definition at line 879 of file dict.cpp.

                                                    {
  if (word.length() == 0) return NO_PERM;
  int i;
  WERD_CHOICE new_word(word.unicharset());
  int last_index = word.length() - 1;
  int new_len = 0;
  for (i = 0; i <= last_index; ++i) {
    UNICHAR_ID unichar_id = (word.unichar_id(i));
    if (getUnicharset().get_ispunctuation(unichar_id)) {
      new_word.append_unichar_id(unichar_id, 1, 0.0, 0.0);
    } else if (!getUnicharset().get_isalpha(unichar_id) &&
               !getUnicharset().get_isdigit(unichar_id)) {
      return false;  // neither punc, nor alpha, nor digit
    } else if ((new_len = new_word.length()) == 0 ||
               new_word.unichar_id(new_len-1) != Dawg::kPatternUnicharID) {
      new_word.append_unichar_id(Dawg::kPatternUnicharID, 1, 0.0, 0.0);
    }
  }
  for (i = 0; i < dawgs_.size(); ++i) {
    if (dawgs_[i] != NULL &&
        dawgs_[i]->type() == DAWG_TYPE_PUNCTUATION &&
        dawgs_[i]->word_in_dawg(new_word)) return true;
  }
  return false;
}

int tesseract::Dict::valid_word	(	const WERD_CHOICE &	word,
		bool	numbers_ok
	)		const

Definition at line 807 of file dict.cpp.

                                                                   {
  const WERD_CHOICE *word_ptr = &word;
  WERD_CHOICE temp_word(word.unicharset());
  if (hyphenated()) {
    copy_hyphen_info(&temp_word);
    temp_word += word;
    word_ptr = &temp_word;
  }
  if (word_ptr->length() == 0) return NO_PERM;
  // Allocate vectors for holding current and updated
  // active_dawgs and constraints and initialize them.
  DawgInfoVector *active_dawgs = new DawgInfoVector[2];
  DawgInfoVector *constraints = new DawgInfoVector[2];
  init_active_dawgs(kAnyWordLength, &(active_dawgs[0]), false);
  init_constraints(&(constraints[0]));
  DawgArgs dawg_args(&(active_dawgs[0]), &(constraints[0]),
                     &(active_dawgs[1]), &(constraints[1]),
                     0.0, NO_PERM, kAnyWordLength, 0);
  int last_index = word_ptr->length() - 1;
  // Call leter_is_okay for each letter in the word.
  for (int i = hyphen_base_size(); i <= last_index; ++i) {
    if (!((this->*letter_is_okay_)(&dawg_args, word_ptr->unichar_id(i),
                                   i == last_index))) break;
    // Swap active_dawgs, constraints with the corresponding updated vector.
    if (dawg_args.updated_active_dawgs == &(active_dawgs[1])) {
      dawg_args.updated_active_dawgs = &(active_dawgs[0]);
      dawg_args.updated_constraints = &(constraints[0]);
      ++(dawg_args.active_dawgs);
      ++(dawg_args.constraints);
    } else {
      ++(dawg_args.updated_active_dawgs);
      ++(dawg_args.updated_constraints);
      dawg_args.active_dawgs = &(active_dawgs[0]);
      dawg_args.constraints = &(constraints[0]);
    }
  }
  delete[] active_dawgs;
  delete[] constraints;
  return valid_word_permuter(dawg_args.permuter, numbers_ok) ?
    dawg_args.permuter : NO_PERM;
}

int tesseract::Dict::valid_word ( const WERD_CHOICE &  word ) const

inline

Definition at line 685 of file dict.h.

                                                {
    return valid_word(word, false);  // return NO_PERM for words with digits
  }

int tesseract::Dict::valid_word ( const char *  string ) const

inline

This function is used by api/tesseract_cube_combiner.cpp.

Definition at line 692 of file dict.h.

                                           {
    WERD_CHOICE word(string, getUnicharset());
    return valid_word(word);
  }

int tesseract::Dict::valid_word_or_number ( const WERD_CHOICE &  word ) const

inline

Definition at line 688 of file dict.h.

                                                          {
    return valid_word(word, true);  // return NUMBER_PERM for valid numbers
  }

static bool tesseract::Dict::valid_word_permuter ( uinT8  perm, bool  numbers_ok  )

inlinestatic

Check all the DAWGs to see if this word is in any of them.

Definition at line 679 of file dict.h.

                                                                      {
    return (perm == SYSTEM_DAWG_PERM || perm == FREQ_DAWG_PERM ||
            perm == DOC_DAWG_PERM || perm == USER_DAWG_PERM ||
            perm == USER_PATTERN_PERM || (numbers_ok && perm == NUMBER_PERM));
  }

void tesseract::Dict::WriteFixedLengthDawgs ( const GenericVector< SquishedDawg * > &  dawg_vec, int  num_dawgs, int  debug_level, FILE *  output_file  )

static

Writes the dawgs in the dawgs_vec to a file. Updates the given table with the indices of dawgs in the dawg_vec for the corresponding word lengths.

Definition at line 626 of file dict.cpp.

                                                       {
  fwrite(&num_dawgs, sizeof(inT32), 1, output_file);
  if (debug_level) tprintf("Writing %d split length dawgs\n", num_dawgs);
  for (int i = 1; i < dawg_vec.size(); ++i) {
    if ((dawg_vec)[i] != NULL) {
      fwrite(&i, sizeof(inT32), 1, output_file);
      dawg_vec[i]->write_squished_dawg(output_file);
      if (debug_level) tprintf("Wrote Dawg with word length %d\n", i);
    }
  }
}

Member Data Documentation

double tesseract::Dict::bestrate_pruning_factor = 2.0

"Multiplying factor of" " current best rate to prune other hypotheses"

Definition at line 874 of file dict.h.

double tesseract::Dict::certainty_scale = 20.0

"Certainty scaling factor"

Definition at line 845 of file dict.h.

int tesseract::Dict::dawg_debug_level = 0

"Set to 1 for general debug info" ", to 2 for more details, to 3 to see all the debug messages"

Definition at line 839 of file dict.h.

double tesseract::Dict::doc_dict_certainty_threshold = -2.25

"Worst certainty" " for words that can be inserted into the document dictionary"

Definition at line 902 of file dict.h.

bool tesseract::Dict::doc_dict_enable = 1

"Enable Document Dictionary "

Definition at line 898 of file dict.h.

double tesseract::Dict::doc_dict_pending_threshold = 0.0

"Worst certainty for using pending dictionary"

Definition at line 900 of file dict.h.

int tesseract::Dict::fragments_debug = 0

"Debug character fragments"

Definition at line 870 of file dict.h.

void(Dict::* tesseract::Dict::go_deeper_fxn_)(const char *debug, const BLOB_CHOICE_LIST_VECTOR &char_choices, int char_choice_index, const CHAR_FRAGMENT_INFO *prev_char_frag_info, bool word_ending, WERD_CHOICE *word, float certainties[], float *limit, WERD_CHOICE *best_choice, int *attempts_left, void *void_more_args)

Pointer to go_deeper function that will be modified by various permuters.

Definition at line 308 of file dict.h.

int tesseract::Dict::hyphen_debug_level = 0

"Debug level for hyphenated words."

Definition at line 840 of file dict.h.

int(Dict::* tesseract::Dict::letter_is_okay_)(void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const

Definition at line 560 of file dict.h.

bool tesseract::Dict::load_bigram_dawg = false

"Load dawg with special word bigrams."

Definition at line 811 of file dict.h.

bool tesseract::Dict::load_fixed_length_dawgs = true

"Load fixed length" " dawgs (e.g. for non-space delimited languages)"

Definition at line 809 of file dict.h.

bool tesseract::Dict::load_freq_dawg = true

"Load frequent word dawg."

Definition at line 803 of file dict.h.

bool tesseract::Dict::load_number_dawg = true

"Load dawg with number patterns."

Definition at line 807 of file dict.h.

bool tesseract::Dict::load_punc_dawg = true

"Load dawg with punctuation patterns."

Definition at line 806 of file dict.h.

bool tesseract::Dict::load_system_dawg = true

"Load system word dawg."

Definition at line 802 of file dict.h.

bool tesseract::Dict::load_unambig_dawg = true

"Load unambiguous word dawg."

Definition at line 804 of file dict.h.

int tesseract::Dict::max_permuter_attempts = 10000

"Maximum number of different" " character choices to consider during permutation." " This limit is especially useful when user patterns" " are specified, since overly generic patterns can result in" " dawg search exploring an overly large number of options."

Definition at line 909 of file dict.h.

int tesseract::Dict::max_viterbi_list_size = 10

"Maximum size of viterbi list."

Definition at line 841 of file dict.h.

bool tesseract::Dict::ngram_permuter_activated = false

"Activate character-level n-gram-based permuter"

Definition at line 904 of file dict.h.

char* tesseract::Dict::output_ambig_words_file = ""

"Output file for ambiguities found in the dictionary"

Definition at line 837 of file dict.h.

bool tesseract::Dict::permute_chartype_word = 0

"Turn on character type (property) consistency permuter"

Definition at line 890 of file dict.h.

bool tesseract::Dict::permute_debug = 0

"Debug char permutation process"

Definition at line 872 of file dict.h.

bool tesseract::Dict::permute_fixed_length_dawg = 0

"Turn on fixed-length phrasebook search permuter"

Definition at line 888 of file dict.h.

bool tesseract::Dict::permute_only_top = false

"Run only the top choice permuter"

Definition at line 910 of file dict.h.

bool tesseract::Dict::permute_script_word = 0

"Turn on word script consistency permuter"

Definition at line 876 of file dict.h.

double(Dict::* tesseract::Dict::probability_in_context_)(const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)

Probability in context function used by the ngram permuter.

Definition at line 570 of file dict.h.

bool tesseract::Dict::save_doc_words = 0

"Save Document Words"

Definition at line 897 of file dict.h.

bool tesseract::Dict::save_raw_choices = false

"Save all explored raw choices"

Definition at line 864 of file dict.h.

int tesseract::Dict::segment_debug = 0

"Debug the whole segmentation process"

Definition at line 871 of file dict.h.

bool tesseract::Dict::segment_nonalphabetic_script = false

"Don't use any alphabetic-specific tricks." "Set to true in the traineddata config file for" " scripts that are cursive or inherently fixed-pitch"

Definition at line 882 of file dict.h.

double tesseract::Dict::segment_penalty_dict_case_bad = 1.3125

"Default score multiplier for word matches, which may have " "case issues (lower is better)."

Definition at line 822 of file dict.h.

double tesseract::Dict::segment_penalty_dict_case_ok = 1.1

"Score multiplier for word matches that have good case " "(lower is better)."

Definition at line 818 of file dict.h.

double tesseract::Dict::segment_penalty_dict_frequent_word = 1.0

"Score multiplier for word matches which have good case and" "are frequent in the given language (lower is better)."

Definition at line 814 of file dict.h.

double tesseract::Dict::segment_penalty_dict_nonword = 1.25

"Score multiplier for glyph fragment segmentations which " "do not match a dictionary word (lower is better)."

Definition at line 830 of file dict.h.

double tesseract::Dict::segment_penalty_garbage = 1.50

"Score multiplier for poorly cased strings that are not in" " the dictionary and generally look like garbage (lower is" " better)."

Definition at line 835 of file dict.h.

double tesseract::Dict::segment_penalty_ngram_best_choice = 1.24

"Multipler to for the best choice from the ngram model."

Definition at line 826 of file dict.h.

double tesseract::Dict::segment_reward_chartype = 0.97

"Score multipler for char type consistency within a word. "

Definition at line 892 of file dict.h.

double tesseract::Dict::segment_reward_ngram_best_choice = 0.99

"Score multipler for ngram permuter's best choice" " (only used in the Han script path)."

Definition at line 896 of file dict.h.

double tesseract::Dict::segment_reward_script = 0.95

"Score multipler for script consistency within a word. " "Being a 'reward' factor, it should be <= 1. " "Smaller value implies bigger reward."

Definition at line 886 of file dict.h.

bool tesseract::Dict::segment_segcost_rating = 0

"incorporate segmentation cost in word rating?"

Definition at line 878 of file dict.h.

double tesseract::Dict::stopper_allowable_character_badness = 3.0

"Max certaintly variation allowed in a word (in sigma)"

Definition at line 855 of file dict.h.

double tesseract::Dict::stopper_ambiguity_threshold_gain = 8.0

"Gain factor for ambiguity threshold."

Definition at line 861 of file dict.h.

double tesseract::Dict::stopper_ambiguity_threshold_offset = 1.5

"Certainty offset for ambiguity threshold."

Definition at line 863 of file dict.h.

double tesseract::Dict::stopper_certainty_per_char = -0.50

"Certainty to add for each dict char above small word size."

Definition at line 853 of file dict.h.

int tesseract::Dict::stopper_debug_level = 0

"Stopper debug level"

Definition at line 856 of file dict.h.

bool tesseract::Dict::stopper_no_acceptable_choices = false

"Make AcceptableChoice() always return false. Useful" " when there is a need to explore all segmentations"

Definition at line 859 of file dict.h.

double tesseract::Dict::stopper_nondict_certainty_base = -2.50

"Certainty threshold for non-dict words"

Definition at line 847 of file dict.h.

double tesseract::Dict::stopper_phase2_certainty_rejection_offset = 1.0

"Reject certainty offset"

Definition at line 849 of file dict.h.

int tesseract::Dict::stopper_smallword_size = 2

"Size of dict word to be treated as non-dict word"

Definition at line 851 of file dict.h.

int tesseract::Dict::tessedit_truncate_wordchoice_log = 10

"Max words to keep in list"

Definition at line 865 of file dict.h.

bool tesseract::Dict::use_only_first_uft8_step = false

"Use only the first UTF8 step of the given string" " when computing log probabilities."

Definition at line 844 of file dict.h.

char* tesseract::Dict::user_patterns_suffix = ""

"A list of user-provided patterns."

Definition at line 801 of file dict.h.

char* tesseract::Dict::user_words_suffix = ""

Variable members. These have to be declared and initialized after image_ptr_, which contains the pointer to the params vector - the member of its base CCUtil class. "A list of user-provided words."

Definition at line 799 of file dict.h.

char* tesseract::Dict::word_to_debug = ""

"Word for which stopper debug information" " should be printed to stdout"

Definition at line 867 of file dict.h.

char* tesseract::Dict::word_to_debug_lengths = ""

"Lengths of unichars in word_to_debug"

Definition at line 869 of file dict.h.

---

The documentation for this class was generated from the following files:

• /home/abuild/rpmbuild/BUILD/tesseract-ocr-3.02.02/dict/dict.h
• /home/abuild/rpmbuild/BUILD/tesseract-ocr-3.02.02/dict/context.cpp
• /home/abuild/rpmbuild/BUILD/tesseract-ocr-3.02.02/dict/dict.cpp
• /home/abuild/rpmbuild/BUILD/tesseract-ocr-3.02.02/dict/hyphen.cpp
• /home/abuild/rpmbuild/BUILD/tesseract-ocr-3.02.02/dict/permdawg.cpp
• /home/abuild/rpmbuild/BUILD/tesseract-ocr-3.02.02/dict/permute.cpp
• /home/abuild/rpmbuild/BUILD/tesseract-ocr-3.02.02/dict/stopper.cpp