diff --git a/python/phi/charclass.py b/python/phi/charclass.py index 6260dd1..717cf86 100644 --- a/python/phi/charclass.py +++ b/python/phi/charclass.py @@ -35,7 +35,14 @@ CHAR_CLASSES = { OPERATOR_CHARS = set("+-*/^%=<>!&|~") BRACKET_CHARS = set("()[]{}") -PUNCT_CHARS = set(".,;:'\"@#$\\") +PUNCT_CHARS = set(".,;:'\"@#$\\_") +WHITESPACE_CHARS = set(" \t\n\r") + +# Unicode mathematical sets (placed at the top to prevent misclassification as standard alpha) +SYMMETRY_CHARS = set("≈≡∼") +PERIODICITY_CHARS = set("πθλ∞") +CONTINUITY_CHARS = set("→Δ∇∫") +META_MATH_CHARS = set("√∑∏∂") def classify_char(c: str) -> int: @@ -84,31 +91,30 @@ def classify_char(c: str) -> int: >>> classify_char("©") 7 """ + # Check Unicode math classes first so Greek/math symbols are not misclassified as standard alpha + if c in PERIODICITY_CHARS: + return 9 + if c in CONTINUITY_CHARS: + return 10 + if c in SYMMETRY_CHARS: + return 8 + if c in META_MATH_CHARS: + return 11 + + # Check standard ASCII classes if c.isdigit(): return 0 if c.isalpha(): return 1 if c.islower() else 2 - # Operator chars - if c in "+-*/^%=<>!&|~": + if c in OPERATOR_CHARS: return 3 - # Bracket chars - if c in "()[]{}": + if c in BRACKET_CHARS: return 4 - # Punctuation - if c in ".,;:'\"@#$\\": + if c in PUNCT_CHARS: return 5 - # Whitespace - if c in (" ", "\t", "\n", "\r"): + if c in WHITESPACE_CHARS: return 6 - # Other (General) - if c in "$\\pi\\theta\\lambda\\infty$": # Periodicity - return 9 - if c in "$\\rightarrow\\Delta\\nabla\\int$": # Continuity - return 10 - if c in "=$\\approx\\equiv\\sim$": # Symmetry - return 8 - if c in "$\\sqrt{}\\sum\\prod\\partial$": # Meta-math - return 11 + return 7