# pyenchant
#
# Copyright (C) 2004-2008, Ryan Kelly
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
#
# In addition, as a special exception, you are
# given permission to link the code of this program with
# non-LGPL Spelling Provider libraries (eg: a MSFT Office
# spell checker backend) and distribute linked combinations including
# the two.  You must obey the GNU Lesser General Public License in all
# respects for all of the code used other than said providers.  If you modify
# this file, you may extend this exception to your version of the
# file, but you are not obligated to do so.  If you do not wish to
# do so, delete this exception statement from your version.
#
"""

    enchant.tokenize.tests:  unittests for enchant tokenization functions.

"""

import unittest
import array

from enchant.tokenize import *
from enchant.tokenize.en import tokenize as tokenize_en
from enchant.utils import raw_unicode, unicode, bytes


class TestTokenization(unittest.TestCase):
    """TestCases for testing the basic tokenization functionality."""

    def test_basic_tokenize(self):
        """Simple regression test for basic white-space tokenization."""
        input = """This is a paragraph.  It's not very special, but it's designed
2 show how the splitter works with many-different combos
of words. Also need to "test" the (handling) of 'quoted' words."""
        output = [
            ("This", 0), ("is", 5), ("a", 8), ("paragraph", 10), ("It's", 22),
            ("not", 27), ("very", 31), ("special", 36), ("but", 45), ("it's", 49),
            ("designed", 54), ("2", 63), ("show", 65), ("how", 70), ("the", 74),
            ("splitter", 78), ("works", 87), ("with", 93), ("many-different", 98),
            ("combos", 113), ("of", 120), ("words", 123),
            ("Also", 130), ("need", 135),
            ("to", 140), ("test", 144), ("the", 150), ("handling", 155),
            ("of", 165), ("quoted", 169), ("words", 177)
        ]
        self.assertEqual(output, [i for i in basic_tokenize(input)])
        for (itmO, itmV) in zip(output, basic_tokenize(input)):
            self.assertEqual(itmO, itmV)

    def test_tokenize_strip(self):
        """Test special-char-stripping edge-cases in basic_tokenize."""
        input = "((' <this> \"\" 'text' has (lots) of (special chars} >>]"
        output = [("<this>", 4), ("text", 15), ("has", 21), ("lots", 26), ("of", 32),
                  ("special", 36), ("chars}", 44), (">>", 51)]
        self.assertEqual(output, [i for i in basic_tokenize(input)])
        for (itmO, itmV) in zip(output, basic_tokenize(input)):
            self.assertEqual(itmO, itmV)

    def test_wrap_tokenizer(self):
        """Test wrapping of one tokenizer with another."""
        input = "this-string will be split@according to diff'rnt rules"
        from enchant.tokenize import en
        tknzr = wrap_tokenizer(basic_tokenize, en.tokenize)
        tknzr = tknzr(input)
        self.assertEqual(tknzr._tokenizer.__class__, basic_tokenize)
        self.assertEqual(tknzr._tokenizer.offset, 0)
        for (n, (word, pos)) in enumerate(tknzr):
            if n == 0:
                self.assertEqual(pos, 0)
                self.assertEqual(word, "this")
            if n == 1:
                self.assertEqual(pos, 5)
                self.assertEqual(word, "string")
            if n == 2:
                self.assertEqual(pos, 12)
                self.assertEqual(word, "will")
                # Test setting offset to a previous token
                tknzr.set_offset(5)
                self.assertEqual(tknzr.offset, 5)
                self.assertEqual(tknzr._tokenizer.offset, 5)
                self.assertEqual(tknzr._curtok.__class__, empty_tokenize)
            if n == 3:
                self.assertEqual(word, "string")
                self.assertEqual(pos, 5)
            if n == 4:
                self.assertEqual(pos, 12)
                self.assertEqual(word, "will")
            if n == 5:
                self.assertEqual(pos, 17)
                self.assertEqual(word, "be")
                # Test setting offset past the current token
                tknzr.set_offset(20)
                self.assertEqual(tknzr.offset, 20)
                self.assertEqual(tknzr._tokenizer.offset, 20)
                self.assertEqual(tknzr._curtok.__class__, empty_tokenize)
            if n == 6:
                self.assertEqual(pos, 20)
                self.assertEqual(word, "split")
            if n == 7:
                self.assertEqual(pos, 26)
                self.assertEqual(word, "according")
                # Test setting offset to middle of current token
                tknzr.set_offset(23)
                self.assertEqual(tknzr.offset, 23)
                self.assertEqual(tknzr._tokenizer.offset, 23)
                self.assertEqual(tknzr._curtok.offset, 3)
            if n == 8:
                self.assertEqual(pos, 23)
                self.assertEqual(word, "it")
                # OK, I'm pretty happy with the behaviour, no need to
                # continue testing the rest of the string


class TestFilters(unittest.TestCase):
    """TestCases for the various Filter subclasses."""

    text = """this text with http://url.com and SomeLinksLike
              ftp://my.site.com.au/some/file AndOthers not:/quite.a.url
              with-an@aemail.address as well"""

    def setUp(self):
        pass

    def test_URLFilter(self):
        """Test filtering of URLs"""
        tkns = get_tokenizer("en_US", filters=(URLFilter,))(self.text)
        out = [t for t in tkns]
        exp = [("this", 0), ("text", 5), ("with", 10), ("and", 30),
               ("SomeLinksLike", 34), ("AndOthers", 93), ("not", 103), ("quite", 108),
               ("a", 114), ("url", 116), ("with", 134), ("an", 139), ("aemail", 142),
               ("address", 149), ("as", 157), ("well", 160)]
        self.assertEqual(out, exp)

    def test_WikiWordFilter(self):
        """Test filtering of WikiWords"""
        tkns = get_tokenizer("en_US", filters=(WikiWordFilter,))(self.text)
        out = [t for t in tkns]
        exp = [("this", 0), ("text", 5), ("with", 10), ("http", 15), ("url", 22), ("com", 26),
               ("and", 30), ("ftp", 62), ("my", 68), ("site", 71), ("com", 76), ("au", 80),
               ("some", 83), ("file", 88), ("not", 103), ("quite", 108),
               ("a", 114), ("url", 116), ("with", 134), ("an", 139), ("aemail", 142),
               ("address", 149), ("as", 157), ("well", 160)]
        self.assertEqual(out, exp)

    def test_EmailFilter(self):
        """Test filtering of email addresses"""
        tkns = get_tokenizer("en_US", filters=(EmailFilter,))(self.text)
        out = [t for t in tkns]
        exp = [("this", 0), ("text", 5), ("with", 10), ("http", 15), ("url", 22), ("com", 26),
               ("and", 30), ("SomeLinksLike", 34),
               ("ftp", 62), ("my", 68), ("site", 71), ("com", 76), ("au", 80),
               ("some", 83), ("file", 88), ("AndOthers", 93), ("not", 103), ("quite", 108),
               ("a", 114), ("url", 116),
               ("as", 157), ("well", 160)]
        self.assertEqual(out, exp)

    def test_CombinedFilter(self):
        """Test several filters combined"""
        tkns = get_tokenizer("en_US", filters=(URLFilter, WikiWordFilter, EmailFilter))(self.text)
        out = [t for t in tkns]
        exp = [("this", 0), ("text", 5), ("with", 10),
               ("and", 30), ("not", 103), ("quite", 108),
               ("a", 114), ("url", 116),
               ("as", 157), ("well", 160)]
        self.assertEqual(out, exp)


class TestChunkers(unittest.TestCase):
    """TestCases for the various Chunker subclasses."""

    def test_HTMLChunker(self):
        """Test filtering of URLs"""
        text = """hello<html><head><title>my title</title></head><body>this is a
                <b>simple</b> HTML document for <p> test<i>ing</i> purposes</p>.
                It < contains > various <-- special characters.
                """
        tkns = get_tokenizer("en_US", chunkers=(HTMLChunker,))(text)
        out = [t for t in tkns]
        exp = [("hello", 0), ("my", 24), ("title", 27), ("this", 53), ("is", 58),
               ("a", 61), ("simple", 82), ("HTML", 93), ("document", 98), ("for", 107),
               ("test", 115), ("ing", 122), ("purposes", 130), ("It", 160),
               ("contains", 165), ("various", 176), ("special", 188),
               ("characters", 196)]
        self.assertEqual(out, exp)
        for (word, pos) in out:
            self.assertEqual(text[pos:pos + len(word)], word)


class TestTokenizeEN(unittest.TestCase):
    """TestCases for checking behaviour of English tokenization."""

    def test_tokenize_en(self):
        """Simple regression test for English tokenization."""
        input = """This is a paragraph.  It's not very special, but it's designed
2 show how the splitter works with many-different combos
of words. Also need to "test" the handling of 'quoted' words."""
        output = [
            ("This", 0), ("is", 5), ("a", 8), ("paragraph", 10), ("It's", 22),
            ("not", 27), ("very", 31), ("special", 36), ("but", 45), ("it's", 49),
            ("designed", 54), ("show", 65), ("how", 70), ("the", 74),
            ("splitter", 78), ("works", 87), ("with", 93), ("many", 98),
            ("different", 103), ("combos", 113), ("of", 120), ("words", 123),
            ("Also", 130), ("need", 135),
            ("to", 140), ("test", 144), ("the", 150), ("handling", 154),
            ("of", 163), ("quoted", 167), ("words", 175)
        ]
        for (itmO, itmV) in zip(output, tokenize_en(input)):
            self.assertEqual(itmO, itmV)

    def test_unicodeBasic(self):
        """Test tokenization of a basic unicode string."""
        input = raw_unicode(
            r"Ik ben ge\u00EFnteresseerd in de co\u00F6rdinatie van mijn knie\u00EBn, maar kan niet \u00E9\u00E9n \u00E0 twee enqu\u00EAtes vinden die recht doet aan mijn carri\u00E8re op Cura\u00E7ao")
        output = input.split(" ")
        output[8] = output[8][0:-1]
        for (itmO, itmV) in zip(output, tokenize_en(input)):
            self.assertEqual(itmO, itmV[0])
            self.assertTrue(input[itmV[1]:].startswith(itmO))

    def test_unicodeCombining(self):
        """Test tokenization with unicode combining symbols."""
        input = raw_unicode(
            r"Ik ben gei\u0308nteresseerd in de co\u00F6rdinatie van mijn knie\u00EBn, maar kan niet e\u0301e\u0301n \u00E0 twee enqu\u00EAtes vinden die recht doet aan mijn carri\u00E8re op Cura\u00E7ao")
        output = input.split(" ")
        output[8] = output[8][0:-1]
        for (itmO, itmV) in zip(output, tokenize_en(input)):
            self.assertEqual(itmO, itmV[0])
            self.assertTrue(input[itmV[1]:].startswith(itmO))

    def test_utf8_bytes(self):
        """Test tokenization of UTF8-encoded bytes (bug #2500184)."""
        # Python3 doesn't support bytestrings, don't run this test
        if str is unicode:
            return
        input = "A r\xc3\xa9sum\xc3\xa9, also spelled resum\xc3\xa9 or resume"
        output = input.split(" ")
        output[1] = output[1][0:-1]
        for (itmO, itmV) in zip(output, tokenize_en(input)):
            self.assertEqual(itmO, itmV[0])
            self.assertTrue(input[itmV[1]:].startswith(itmO))

    def test_utf8_bytes_at_end(self):
        """Test tokenization of UTF8-encoded bytes at end of word."""
        # Python3 doesn't support bytestrings, don't run this test
        if str is unicode:
            return
        input = "A r\xc3\xa9sum\xc3\xa9, also spelled resum\xc3\xa9 or resume"
        output = input.split(" ")
        output[1] = output[1][0:-1]
        for (itmO, itmV) in zip(output, tokenize_en(input)):
            self.assertEqual(itmO, itmV[0])

    def test_utf8_bytes_in_an_array(self):
        """Test tokenization of UTF8-encoded bytes stored in an array."""
        # Python3 doesn't support bytestrings, don't run this test
        if str is unicode:
            return
        input = "A r\xc3\xa9sum\xc3\xa9, also spelled resum\xc3\xa9 or resume"
        output = input.split(" ")
        output[1] = output[1][0:-1]
        input = array.array('c', input)
        output = [array.array('c', w) for w in output]
        for (itmO, itmV) in zip(output, tokenize_en(array.array('c', input))):
            self.assertEqual(itmO, itmV[0])
            self.assertEqual(input[itmV[1]:itmV[1] + len(itmV[0])], itmO)

    def test_bug1591450(self):
        """Check for tokenization regressions identified in bug #1591450."""
        input = """Testing <i>markup</i> and {y:i}so-forth...leading dots and trail--- well, you get-the-point. Also check numbers: 999 1,000 12:00 .45. Done?"""
        output = [
            ("Testing", 0), ("i", 9), ("markup", 11), ("i", 19), ("and", 22),
            ("y", 27), ("i", 29), ("so", 31), ("forth", 34), ("leading", 42),
            ("dots", 50), ("and", 55), ("trail", 59), ("well", 68),
            ("you", 74), ("get", 78), ("the", 82), ("point", 86),
            ("Also", 93), ("check", 98), ("numbers", 104), ("Done", 134),
        ]
        for (itmO, itmV) in zip(output, tokenize_en(input)):
            self.assertEqual(itmO, itmV)

    def test_bug2785373(self):
        """Testcases for bug #2785373"""
        input = "So, one dey when I wes 17, I left."
        for _ in tokenize_en(input):
            pass
        input = raw_unicode("So, one dey when I wes 17, I left.")
        for _ in tokenize_en(input):
            pass

    def test_finnish_text(self):
        """Test tokenizing some Finnish text.

        This really should work since there are no special rules to apply,
        just lots of non-ascii characters.
        """
        inputT = raw_unicode(
            'T\\xe4m\\xe4 on kappale. Eip\\xe4 ole kovin 2 nen, mutta tarkoitus on n\\xe4ytt\\xe4\\xe4 miten sanastaja \\ntoimii useiden-erilaisten sanarypp\\xe4iden kimpussa.\\nPit\\xe4\\xe4p\\xe4 viel\\xe4 \'tarkistaa\' sanat jotka "lainausmerkeiss\\xe4". Heittomerkki ja vaa\'an.\\nUlkomaisia sanoja s\\xfcss, spa\\xdf.')
        outputT = [
            (raw_unicode('T\\xe4m\\xe4'), 0), (raw_unicode('on'), 5), (raw_unicode('kappale'), 8),
            (raw_unicode('Eip\\xe4'), 17), (raw_unicode('ole'), 22), (raw_unicode('kovin'), 26),
            (raw_unicode('nen'), 34), (raw_unicode('mutta'), 39), (raw_unicode('tarkoitus'), 45),
            (raw_unicode('on'), 55), (raw_unicode('n\\xe4ytt\\xe4\\xe4'), 58), (raw_unicode('miten'), 66),
            (raw_unicode('sanastaja'), 72), (raw_unicode('toimii'), 83), (raw_unicode('useiden'), 90),
            (raw_unicode('erilaisten'), 98), (raw_unicode('sanarypp\\xe4iden'), 109), (raw_unicode('kimpussa'), 123),
            (raw_unicode('Pit\\xe4\\xe4p\\xe4'), 133), (raw_unicode('viel\\xe4'), 141), (raw_unicode('tarkistaa'), 148),
            (raw_unicode('sanat'), 159), (raw_unicode('jotka'), 165), (raw_unicode('lainausmerkeiss\\xe4'), 172),
            (raw_unicode('Heittomerkki'), 191), (raw_unicode('ja'), 204), (raw_unicode("vaa'an"), 207),
            (raw_unicode('Ulkomaisia'), 215), (raw_unicode('sanoja'), 226), (raw_unicode('s\\xfcss'), 233),
            (raw_unicode('spa\\xdf'), 239), ]
        for (itmO, itmV) in zip(outputT, tokenize_en(inputT)):
            self.assertEqual(itmO, itmV)