From a3e2a361e11f72a095d0584c7078ccbb2a94f2a0 Mon Sep 17 00:00:00 2001
From: Nikolay Pultsin <geometer@fbreader.org>
Date: Sun, 1 Apr 2012 20:41:39 +0100
Subject: [PATCH] Tamil language detection patterns

---
 assets/languagePatterns/ta_utf-16   | 75 ++++++++++++++++++++++++++
 assets/languagePatterns/ta_utf-16be | 84 +++++++++++++++++++++++++++++
 assets/languagePatterns/ta_utf-8    | 67 +++++++++++++++++++++++
 3 files changed, 226 insertions(+)
 create mode 100644 assets/languagePatterns/ta_utf-16
 create mode 100644 assets/languagePatterns/ta_utf-16be
 create mode 100644 assets/languagePatterns/ta_utf-8

diff --git a/assets/languagePatterns/ta_utf-16 b/assets/languagePatterns/ta_utf-16
new file mode 100644
index 000000000..42723c6d1
--- /dev/null
+++ b/assets/languagePatterns/ta_utf-16
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<statistics charSequenceSize="2" size="72" volume="71501" squaresVolume="163982665">
+  <item sequence="0x00 0x92" frequency="47"/>
+  <item sequence="0x00 0x95" frequency="530"/>
+  <item sequence="0x00 0x9a" frequency="301"/>
+  <item sequence="0x00 0xa4" frequency="544"/>
+  <item sequence="0x00 0xa8" frequency="316"/>
+  <item sequence="0x00 0xaa" frequency="669"/>
+  <item sequence="0x00 0xae" frequency="419"/>
+  <item sequence="0x00 0xaf" frequency="32"/>
+  <item sequence="0x00 0xb0" frequency="23"/>
+  <item sequence="0x00 0xb2" frequency="15"/>
+  <item sequence="0x00 0xb5" frequency="340"/>
+  <item sequence="0x0b 0x95" frequency="1928"/>
+  <item sequence="0x0b 0x99" frequency="373"/>
+  <item sequence="0x0b 0x9a" frequency="536"/>
+  <item sequence="0x0b 0x9e" frequency="127"/>
+  <item sequence="0x0b 0x9f" frequency="1294"/>
+  <item sequence="0x0b 0xa3" frequency="464"/>
+  <item sequence="0x0b 0xa4" frequency="1798"/>
+  <item sequence="0x0b 0xa8" frequency="443"/>
+  <item sequence="0x0b 0xa9" frequency="1433"/>
+  <item sequence="0x0b 0xaa" frequency="884"/>
+  <item sequence="0x0b 0xae" frequency="1316"/>
+  <item sequence="0x0b 0xaf" frequency="1008"/>
+  <item sequence="0x0b 0xb0" frequency="1972"/>
+  <item sequence="0x0b 0xb1" frequency="771"/>
+  <item sequence="0x0b 0xb2" frequency="1070"/>
+  <item sequence="0x0b 0xb3" frequency="891"/>
+  <item sequence="0x0b 0xb4" frequency="313"/>
+  <item sequence="0x0b 0xb5" frequency="733"/>
+  <item sequence="0x0b 0xbe" frequency="1567"/>
+  <item sequence="0x0b 0xbf" frequency="2364"/>
+  <item sequence="0x0b 0xc0" frequency="192"/>
+  <item sequence="0x0b 0xc1" frequency="2434"/>
+  <item sequence="0x0b 0xc2" frequency="240"/>
+  <item sequence="0x0b 0xc6" frequency="323"/>
+  <item sequence="0x0b 0xc7" frequency="357"/>
+  <item sequence="0x0b 0xc8" frequency="829"/>
+  <item sequence="0x0b 0xca" frequency="260"/>
+  <item sequence="0x0b 0xcb" frequency="284"/>
+  <item sequence="0x0b 0xcd" frequency="6276"/>
+  <item sequence="0x92 0x0b" frequency="49"/>
+  <item sequence="0x95 0x0b" frequency="2460"/>
+  <item sequence="0x99 0x0b" frequency="373"/>
+  <item sequence="0x9a 0x0b" frequency="839"/>
+  <item sequence="0x9e 0x0b" frequency="132"/>
+  <item sequence="0x9f 0x0b" frequency="1307"/>
+  <item sequence="0xa3 0x0b" frequency="466"/>
+  <item sequence="0xa4 0x0b" frequency="2342"/>
+  <item sequence="0xa8 0x0b" frequency="762"/>
+  <item sequence="0xa9 0x0b" frequency="1440"/>
+  <item sequence="0xaa 0x0b" frequency="1553"/>
+  <item sequence="0xae 0x0b" frequency="1736"/>
+  <item sequence="0xaf 0x0b" frequency="1040"/>
+  <item sequence="0xb0 0x0b" frequency="1995"/>
+  <item sequence="0xb1 0x0b" frequency="792"/>
+  <item sequence="0xb2 0x0b" frequency="1085"/>
+  <item sequence="0xb3 0x0b" frequency="893"/>
+  <item sequence="0xb4 0x0b" frequency="313"/>
+  <item sequence="0xb5 0x0b" frequency="1073"/>
+  <item sequence="0xbe 0x0b" frequency="1567"/>
+  <item sequence="0xbf 0x0b" frequency="2364"/>
+  <item sequence="0xc0 0x0b" frequency="192"/>
+  <item sequence="0xc1 0x0b" frequency="2434"/>
+  <item sequence="0xc2 0x0b" frequency="240"/>
+  <item sequence="0xc6 0x0b" frequency="323"/>
+  <item sequence="0xc7 0x0b" frequency="357"/>
+  <item sequence="0xc8 0x0b" frequency="829"/>
+  <item sequence="0xca 0x0b" frequency="260"/>
+  <item sequence="0xcb 0x0b" frequency="284"/>
+  <item sequence="0xcd 0x0b" frequency="6276"/>
+  <item sequence="0xfe 0xff" frequency="3"/>
+  <item sequence="0xff 0xfe" frequency="6"/>
+</statistics>
diff --git a/assets/languagePatterns/ta_utf-16be b/assets/languagePatterns/ta_utf-16be
new file mode 100644
index 000000000..d134993b9
--- /dev/null
+++ b/assets/languagePatterns/ta_utf-16be
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<statistics charSequenceSize="2" size="81" volume="71398" squaresVolume="145939184">
+  <item sequence="0x0b 0x92" frequency="49"/>
+  <item sequence="0x0b 0x95" frequency="2460"/>
+  <item sequence="0x0b 0x99" frequency="373"/>
+  <item sequence="0x0b 0x9a" frequency="839"/>
+  <item sequence="0x0b 0x9e" frequency="132"/>
+  <item sequence="0x0b 0x9f" frequency="1307"/>
+  <item sequence="0x0b 0xa3" frequency="466"/>
+  <item sequence="0x0b 0xa4" frequency="2342"/>
+  <item sequence="0x0b 0xa8" frequency="762"/>
+  <item sequence="0x0b 0xa9" frequency="1440"/>
+  <item sequence="0x0b 0xaa" frequency="1553"/>
+  <item sequence="0x0b 0xae" frequency="1736"/>
+  <item sequence="0x0b 0xaf" frequency="1040"/>
+  <item sequence="0x0b 0xb0" frequency="1995"/>
+  <item sequence="0x0b 0xb1" frequency="792"/>
+  <item sequence="0x0b 0xb2" frequency="1085"/>
+  <item sequence="0x0b 0xb3" frequency="893"/>
+  <item sequence="0x0b 0xb4" frequency="313"/>
+  <item sequence="0x0b 0xb5" frequency="1073"/>
+  <item sequence="0x0b 0xbe" frequency="1567"/>
+  <item sequence="0x0b 0xbf" frequency="2364"/>
+  <item sequence="0x0b 0xc0" frequency="192"/>
+  <item sequence="0x0b 0xc1" frequency="2434"/>
+  <item sequence="0x0b 0xc2" frequency="240"/>
+  <item sequence="0x0b 0xc6" frequency="323"/>
+  <item sequence="0x0b 0xc7" frequency="357"/>
+  <item sequence="0x0b 0xc8" frequency="829"/>
+  <item sequence="0x0b 0xca" frequency="260"/>
+  <item sequence="0x0b 0xcb" frequency="284"/>
+  <item sequence="0x0b 0xcd" frequency="6276"/>
+  <item sequence="0x92 0x0b" frequency="49"/>
+  <item sequence="0x95 0x00" frequency="163"/>
+  <item sequence="0x95 0x0b" frequency="2296"/>
+  <item sequence="0x99 0x0b" frequency="373"/>
+  <item sequence="0x9a 0x0b" frequency="836"/>
+  <item sequence="0x9e 0x0b" frequency="132"/>
+  <item sequence="0x9f 0x0b" frequency="1185"/>
+  <item sequence="0xa3 0x0b" frequency="464"/>
+  <item sequence="0xa4 0x00" frequency="160"/>
+  <item sequence="0xa4 0x0b" frequency="2182"/>
+  <item sequence="0xa8 0x0b" frequency="762"/>
+  <item sequence="0xa9 0x00" frequency="102"/>
+  <item sequence="0xa9 0x0b" frequency="1338"/>
+  <item sequence="0xaa 0x00" frequency="6"/>
+  <item sequence="0xaa 0x0b" frequency="1547"/>
+  <item sequence="0xae 0x00" frequency="3"/>
+  <item sequence="0xae 0x0b" frequency="1733"/>
+  <item sequence="0xaf 0x00" frequency="130"/>
+  <item sequence="0xaf 0x0b" frequency="909"/>
+  <item sequence="0xb0 0x0b" frequency="1982"/>
+  <item sequence="0xb1 0x00" frequency="59"/>
+  <item sequence="0xb1 0x0b" frequency="733"/>
+  <item sequence="0xb2 0x00" frequency="20"/>
+  <item sequence="0xb2 0x0b" frequency="1065"/>
+  <item sequence="0xb3 0x00" frequency="48"/>
+  <item sequence="0xb3 0x0b" frequency="845"/>
+  <item sequence="0xb4 0x00" frequency="9"/>
+  <item sequence="0xb4 0x0b" frequency="304"/>
+  <item sequence="0xb5 0x00" frequency="11"/>
+  <item sequence="0xb5 0x0b" frequency="1062"/>
+  <item sequence="0xbe 0x00" frequency="84"/>
+  <item sequence="0xbe 0x0b" frequency="1483"/>
+  <item sequence="0xbf 0x00" frequency="290"/>
+  <item sequence="0xbf 0x0b" frequency="2074"/>
+  <item sequence="0xc0 0x00" frequency="20"/>
+  <item sequence="0xc0 0x0b" frequency="172"/>
+  <item sequence="0xc1 0x00" frequency="572"/>
+  <item sequence="0xc1 0x0b" frequency="1859"/>
+  <item sequence="0xc2 0x0b" frequency="229"/>
+  <item sequence="0xc6 0x0b" frequency="323"/>
+  <item sequence="0xc7 0x00" frequency="143"/>
+  <item sequence="0xc7 0x0b" frequency="214"/>
+  <item sequence="0xc8 0x00" frequency="268"/>
+  <item sequence="0xc8 0x0b" frequency="561"/>
+  <item sequence="0xca 0x0b" frequency="260"/>
+  <item sequence="0xcb 0x00" frequency="10"/>
+  <item sequence="0xcb 0x0b" frequency="274"/>
+  <item sequence="0xcd 0x00" frequency="2610"/>
+  <item sequence="0xcd 0x0b" frequency="3662"/>
+  <item sequence="0xfe 0xff" frequency="3"/>
+  <item sequence="0xff 0x0b" frequency="3"/>
+</statistics>
diff --git a/assets/languagePatterns/ta_utf-8 b/assets/languagePatterns/ta_utf-8
new file mode 100644
index 000000000..d37dbc7c4
--- /dev/null
+++ b/assets/languagePatterns/ta_utf-8
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<statistics charSequenceSize="2" size="64" volume="104788" squaresVolume="952469468">
+  <item sequence="0x80 0xe0" frequency="172"/>
+  <item sequence="0x81 0xe0" frequency="1859"/>
+  <item sequence="0x82 0xe0" frequency="229"/>
+  <item sequence="0x86 0xe0" frequency="604"/>
+  <item sequence="0x87 0xe0" frequency="449"/>
+  <item sequence="0x88 0xe0" frequency="565"/>
+  <item sequence="0x8a 0xe0" frequency="407"/>
+  <item sequence="0x8b 0xe0" frequency="274"/>
+  <item sequence="0x8d 0xe0" frequency="3662"/>
+  <item sequence="0x92 0xe0" frequency="49"/>
+  <item sequence="0x95 0xe0" frequency="2296"/>
+  <item sequence="0x99 0xe0" frequency="373"/>
+  <item sequence="0x9a 0xe0" frequency="836"/>
+  <item sequence="0x9e 0xe0" frequency="132"/>
+  <item sequence="0x9f 0xe0" frequency="1185"/>
+  <item sequence="0xa3 0xe0" frequency="464"/>
+  <item sequence="0xa4 0xe0" frequency="2182"/>
+  <item sequence="0xa8 0xe0" frequency="762"/>
+  <item sequence="0xa9 0xe0" frequency="1338"/>
+  <item sequence="0xaa 0xe0" frequency="1547"/>
+  <item sequence="0xae 0x92" frequency="49"/>
+  <item sequence="0xae 0x95" frequency="2460"/>
+  <item sequence="0xae 0x99" frequency="373"/>
+  <item sequence="0xae 0x9a" frequency="839"/>
+  <item sequence="0xae 0x9e" frequency="132"/>
+  <item sequence="0xae 0x9f" frequency="1307"/>
+  <item sequence="0xae 0xa3" frequency="466"/>
+  <item sequence="0xae 0xa4" frequency="2342"/>
+  <item sequence="0xae 0xa8" frequency="762"/>
+  <item sequence="0xae 0xa9" frequency="1440"/>
+  <item sequence="0xae 0xaa" frequency="1553"/>
+  <item sequence="0xae 0xae" frequency="1736"/>
+  <item sequence="0xae 0xaf" frequency="1040"/>
+  <item sequence="0xae 0xb0" frequency="1995"/>
+  <item sequence="0xae 0xb1" frequency="792"/>
+  <item sequence="0xae 0xb2" frequency="1085"/>
+  <item sequence="0xae 0xb3" frequency="893"/>
+  <item sequence="0xae 0xb4" frequency="313"/>
+  <item sequence="0xae 0xb5" frequency="1073"/>
+  <item sequence="0xae 0xbe" frequency="1567"/>
+  <item sequence="0xae 0xbf" frequency="2364"/>
+  <item sequence="0xae 0xe0" frequency="1733"/>
+  <item sequence="0xaf 0x80" frequency="192"/>
+  <item sequence="0xaf 0x81" frequency="2434"/>
+  <item sequence="0xaf 0x82" frequency="240"/>
+  <item sequence="0xaf 0x86" frequency="323"/>
+  <item sequence="0xaf 0x87" frequency="357"/>
+  <item sequence="0xaf 0x88" frequency="829"/>
+  <item sequence="0xaf 0x8a" frequency="260"/>
+  <item sequence="0xaf 0x8b" frequency="284"/>
+  <item sequence="0xaf 0x8d" frequency="6276"/>
+  <item sequence="0xaf 0xe0" frequency="909"/>
+  <item sequence="0xb0 0xe0" frequency="1982"/>
+  <item sequence="0xb1 0xe0" frequency="733"/>
+  <item sequence="0xb2 0xe0" frequency="1065"/>
+  <item sequence="0xb3 0xe0" frequency="845"/>
+  <item sequence="0xb4 0xe0" frequency="304"/>
+  <item sequence="0xb5 0xe0" frequency="1062"/>
+  <item sequence="0xbb 0xbf" frequency="3"/>
+  <item sequence="0xbe 0xe0" frequency="1483"/>
+  <item sequence="0xbf 0xe0" frequency="2077"/>
+  <item sequence="0xe0 0xae" frequency="26230"/>
+  <item sequence="0xe0 0xaf" frequency="11198"/>
+  <item sequence="0xef 0xbb" frequency="3"/>
+</statistics>