Skip to content

Commit b42e12f

Browse files
HugoFaraHugoFara
committed
First step to repair long_text_import.
Texts are now imported but Japanese is ill-parsed.
1 parent 59f8795 commit b42e12f

2 files changed

Lines changed: 278 additions & 209 deletions

File tree

‎inc/database_connect.php‎

Lines changed: 101 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,14 @@ function get_first_value($sql)
9191
return $d;
9292
}
9393

94-
// -------------------------------------------------------------
9594

95+
/**
96+
* Replace Windows line return ("\r\n") by Linux ones ("\n").
97+
*
98+
* @param string $s Input string
99+
*
100+
* @return string Adapted string.
101+
*/
96102
function prepare_textdata($s): string
97103
{
98104
return str_replace("\r\n", "\n", $s);
@@ -639,15 +645,14 @@ function set_word_count(): void
639645
while($rec = mysqli_fetch_assoc($result)){
640646
if ($rec['LgSplitEachChar']) {
641647
$textlc = preg_replace('/([^\s])/u', "$1 ", $rec['WoTextLC']);
642-
}
643-
else{
648+
} else {
644649
$textlc = $rec['WoTextLC'];
645650
}
646651
$sqlarr[]= ' WHEN ' . $rec['WoID'] . '
647652
THEN ' . preg_match_all(
648653
'/([' . $rec['LgRegexpWordCharacters'] . ']+)/u', $textlc, $ma
649654
);
650-
if(++$i % 1000 == 0) {
655+
if (++$i % 1000 == 0) {
651656
//if(!empty($sqlarr)) { cannot be empty
652657
$max = $rec['WoID'];
653658
$sqltext = "UPDATE " . $tbpref . "words SET WoWordCount = CASE WoID";
@@ -660,7 +665,7 @@ function set_word_count(): void
660665
}
661666
}
662667
mysqli_free_result($result);
663-
if(!empty($sqlarr)) {
668+
if (!empty($sqlarr)) {
664669
$sqltext = "UPDATE " . $tbpref . "words SET WoWordCount = CASE WoID";
665670
$sqltext .= implode(' ', $sqlarr) . ' END where WoWordCount=0';
666671
do_mysqli_query($sqltext);
@@ -670,29 +675,34 @@ function set_word_count(): void
670675
/**
671676
* Parse a Japanese text using MeCab and add it to the database.
672677
*
673-
* @param {string} $text Text to parse.
674-
* @param {int} $id Text ID
678+
* @param string $text Text to parse.
679+
* @param int $id Text ID. If $id = -1 print results,
680+
* if $id = -2 return splitted texts
675681
*
676-
* @return void
682+
* @return string[]|void Splitted sentence if $id = -2
683+
*
684+
* @global string $tbpref Database table prefix
677685
*/
678686
function parse_japanese_text($text, $id) {
679687
global $tbpref;
680688
$file_name = tempnam(sys_get_temp_dir(), $tbpref . "tmpti");
681-
// Ze use the format "zord num num" for all nodes
689+
// We use the format "word num num" for all nodes
682690
$mecab_args = " -F %m\\t%t\\t%h\\n -U %m\\t%t\\t%h\\n -E EOS\\t3\\t7\\n";
683691
$mecab_args .= " -o $file_name ";
684692
$mecab = get_mecab_path($mecab_args);
685-
$s = preg_replace('/[ \t]+/u', ' ', $text);
686-
$s = trim($s);
687-
if ($id == -1) {
693+
$text = preg_replace('/[ \t]+/u', ' ', $text);
694+
$text = trim($text);
695+
if ($id == -1) {
688696
echo '<div id="check_text" style="margin-right:50px;">
689697
<h4>Text</h4>
690-
<p>' . str_replace("\n", "<br /><br />", tohtml($s)). '</p>';
698+
<p>' . str_replace("\n", "<br /><br />", tohtml($text)). '</p>';
699+
} else if ($id == -2) {
700+
return explode("\n", $text);
691701
}
692702

693703
// WARNING: \n is converted to PHP_EOL here!
694704
$handle = popen($mecab, 'w');
695-
fwrite($handle, $s);
705+
fwrite($handle, $text);
696706
pclose($handle);
697707

698708
runsql(
@@ -758,11 +768,13 @@ function parse_japanese_text($text, $id) {
758768
/**
759769
* Parse a text using the default tools. It is a not-japanese text.
760770
*
761-
* @param {string} $text Text to parse
762-
* @param {int} $id Text ID
763-
* @param {int} $lid Language ID
771+
* @param string $text Text to parse
772+
* @param int $id Text ID. If $id == -2, only split the text.
773+
* @param int $lid Language ID.
764774
*
765-
* @return void
775+
* @return void|string[] If $id == -2 return a splitted version of the text.
776+
*
777+
* @global string $tbpref Database table prefix
766778
*/
767779
function parse_standard_text($text, $id, $lid) {
768780
global $tbpref;
@@ -774,47 +786,46 @@ function parse_standard_text($text, $id, $lid) {
774786
$noSentenceEnd = $record['LgExceptionsSplitSentences'];
775787
$termchar = $record['LgRegexpWordCharacters'];
776788
$rtlScript = $record['LgRightToLeft'];
777-
$s = $text;
778-
$s = str_replace("\n", "", $s);
779-
$s = trim($s);
789+
$s = str_replace("\n", "", $text);
790+
$text = trim($text);
780791
if ($record['LgSplitEachChar']) {
781-
$s = preg_replace('/([^\s])/u', "$1\t", $s);
792+
$text = preg_replace('/([^\s])/u', "$1\t", $text);
782793
}
783-
$s = preg_replace('/\s+/u', ' ', $s);
794+
$text = preg_replace('/\s+/u', ' ', $text);
784795
if ($id == -1) {
785796
echo "<div id=\"check_text\" style=\"margin-right:50px;\">
786797
<h4>Text</h4>
787798
<p " . ($rtlScript ? 'dir="rtl"' : '') . ">" .
788-
str_replace("", "<br /><br />", tohtml($s)) .
799+
str_replace("", "<br /><br />", tohtml($text)) .
789800
"</p>";
790801
}
791802
// "\r" => Sentence delimiter, "\t" and "\n" => Word delimiter
792-
$s = preg_replace_callback(
803+
$text = preg_replace_callback(
793804
"/(\S+)\s*((\.+)|([$splitSentence]))([]'`\"”)‘’‹›“„«»』」]*)(?=(\s*)(\S+|$))/u",
794805
// Arrow functions got introduced in PHP 7.4
795806
//fn ($matches) => find_latin_sentence_end($matches, $noSentenceEnd)
796807
function ($matches) use ($noSentenceEnd) {
797808
return find_latin_sentence_end($matches, $noSentenceEnd);
798809
},
799-
$s
810+
$text
800811
);
801-
$s = str_replace(array("",""), array("\r","\r"), $s);
802-
$s = preg_replace(
812+
$text = str_replace(array("",""), array("\r","\r"), $text);
813+
$text = preg_replace(
803814
array(
804815
'/([^' . $termchar . '])/u',
805816
'/\n([' . $splitSentence . '][\'`"”)\]‘’‹›“„«»』」]*)\n\t/u',
806817
'/([0-9])[\n]([:.,])[\n]([0-9])/u'
807818
),
808819
array("\n$1\n", "$1", "$1$2$3"),
809-
$s
820+
$text
810821
);
811822
if ($id == -2) {
812823
return explode("\r", remove_spaces(
813824
str_replace(
814-
array("\r\r","\t","\n"), array("\r","",""), $s
825+
array("\r\r","\t","\n"), array("\r","",""), $text
815826
),
816-
$removeSpaces)
817-
);
827+
$removeSpaces
828+
));
818829
}
819830

820831

@@ -877,11 +888,13 @@ function ($matches) use ($noSentenceEnd) {
877888
/**
878889
* Pre-parse the input text before a definitive parsing by a specialized parser.
879890
*
880-
* @param {string} $text Text to parse
881-
* @param {int} $id Text ID
882-
* @param {int} $lid Language ID
891+
* @param string $text Text to parse
892+
* @param int $id Text ID
893+
* @param int $lid Language ID
883894
*
884-
* @return void
895+
* @return string[]|void If $id = -2 return a splitted version of the text
896+
*
897+
* @global string $tbpref Database table prefix
885898
*/
886899
function prepare_text_parsing($text, $id, $lid) {
887900
global $tbpref;
@@ -891,24 +904,23 @@ function prepare_text_parsing($text, $id, $lid) {
891904
$termchar = $record['LgRegexpWordCharacters'];
892905
$replace = explode("|", $record['LgCharacterSubstitutions']);
893906
mysqli_free_result($res);
894-
$s = prepare_textdata($text);
907+
$text = prepare_textdata($text);
895908
//if(is_callable('normalizer_normalize')) $s = normalizer_normalize($s);
896909
do_mysqli_query('TRUNCATE TABLE ' . $tbpref . 'temptextitems');
897910

898911
// because of sentence special characters
899-
$s = str_replace(array('}','{'), array(']','['), $s);
912+
$text = str_replace(array('}','{'), array(']','['), $text);
900913
foreach ($replace as $value) {
901914
$fromto = explode("=", trim($value));
902-
if(count($fromto) >= 2) {
903-
$s = str_replace(trim($fromto[0]), trim($fromto[1]), $s);
915+
if (count($fromto) >= 2) {
916+
$text = str_replace(trim($fromto[0]), trim($fromto[1]), $text);
904917
}
905918
}
906919

907920
if ('MECAB'== strtoupper(trim($termchar))) {
908-
parse_japanese_text($text, $id);
909-
} else {
910-
parse_standard_text($text, $id, $lid);
911-
}
921+
return parse_japanese_text($text, $id);
922+
}
923+
return parse_standard_text($text, $id, $lid);
912924
}
913925

914926
/**
@@ -936,19 +948,18 @@ function check_text_valid($lid)
936948
'SELECT count(`TiOrder`) cnt, if(0=TiWordCount,0,1) as len,
937949
LOWER(TiText) as word, WoTranslation
938950
FROM ' . $tbpref . 'temptextitems
939-
left join ' . $tbpref . 'words on lower(TiText)=WoTextLC and WoLgID=' . $lid . '
940-
group by lower(TiText)'
951+
LEFT JOIN ' . $tbpref . 'words ON lower(TiText)=WoTextLC AND WoLgID=' . $lid . '
952+
GROUP BY lower(TiText)'
941953
);
942-
while($record = mysqli_fetch_assoc($res)){
943-
if($record['len']==1) {
954+
while ($record = mysqli_fetch_assoc($res)) {
955+
if ($record['len']==1) {
944956
$wo[]= array(
945957
tohtml($record['word']),
946958
$record['cnt'],
947959
tohtml($record['WoTranslation'])
948960
);
949-
}
950-
else{
951-
$nw[]= array(tohtml($record['word']),tohtml($record['cnt']));
961+
} else{
962+
$nw[] = array(tohtml($record['word']), tohtml($record['cnt']));
952963
}
953964
}
954965
mysqli_free_result($res);
@@ -962,11 +973,13 @@ function check_text_valid($lid)
962973
/**
963974
* Change the default values for default language, default text, etc...
964975
*
965-
* @param {int} $id New default text ID
966-
* @param {int} $lid New default language ID
967-
* @param {string} $sql
976+
* @param int $id New default text ID
977+
* @param int $lid New default language ID
978+
* @param string $sql
968979
*
969980
* @return void
981+
*
982+
* @global string $tbpref Database table prefix
970983
*/
971984
function update_default_values($id, $lid, $sql) {
972985
global $tbpref;
@@ -1016,9 +1029,9 @@ function update_default_values($id, $lid, $sql) {
10161029
/**
10171030
* Check a text and display statistics about it.
10181031
*
1019-
* @param {string} $sql
1020-
* @param {bool} $rtlScript
1021-
* @param {} $wl
1032+
* @param string $sql
1033+
* @param bool $rtlScript true if language is right-to-left
1034+
* @param string[] $wl Words lengths
10221035
*
10231036
* @return void
10241037
*/
@@ -1097,7 +1110,9 @@ function check_text_with_expressions($id, $lid, $wl, $wl_max, $mw_sql)
10971110
$init_var = '@a' . strval($i) . '=0,';
10981111
}
10991112
do_mysqli_query('set ' . $init_var . '@a1=0,@a0=0,@b=0,@c="",@d=0,@e=0,@f="",@h=0;');
1100-
do_mysqli_query('CREATE TEMPORARY TABLE IF NOT EXISTS ' . $tbpref . 'numbers( n tinyint(3) unsigned NOT NULL);');
1113+
do_mysqli_query(
1114+
'CREATE TEMPORARY TABLE IF NOT EXISTS ' . $tbpref . 'numbers( n tinyint(3) unsigned NOT NULL);'
1115+
);
11011116
do_mysqli_query('TRUNCATE TABLE ' . $tbpref . 'numbers');
11021117
do_mysqli_query('INSERT IGNORE INTO ' . $tbpref . 'numbers(n) VALUES (' . implode('),(', $wl) . ');');
11031118
if ($id>0) {
@@ -1173,7 +1188,7 @@ function check_text_with_expressions($id, $lid, $wl, $wl_max, $mw_sql)
11731188
*
11741189
* @global string $tbpref Database table prefix
11751190
*
1176-
* @return null|string[]
1191+
* @return null|string[] The sentence array if $id = -2
11771192
*
11781193
* @psalm-return non-empty-list<string>|null
11791194
*/
@@ -1193,37 +1208,51 @@ function splitCheckText($text, $lid, $id)
11931208
$rtlScript = $record['LgRightToLeft'];
11941209
mysqli_free_result($res);
11951210

1211+
if ($id == -2) {
1212+
/*
1213+
Replacement code not created yet
1214+
1215+
trigger_error(
1216+
"Using splitCheckText with \$id == -2 is deprectad and won't work in
1217+
LWT 3.0.0. Use format_text instead.",
1218+
E_USER_WARNING
1219+
);*/
1220+
return prepare_text_parsing($text, -2, $lid);
1221+
}
11961222
prepare_text_parsing($text, $id, $lid);
11971223

1198-
if ($id==-1) {//check text
1224+
// Check text
1225+
if ($id == -1) {
11991226
check_text_valid($lid);
1200-
}//check text end
1227+
}
12011228

12021229
$res = do_mysqli_query(
1203-
"SELECT WoWordCount AS len, count(WoWordCount) AS cnt
1230+
"SELECT WoWordCount AS word_count, count(WoWordCount) AS cnt
12041231
FROM " . $tbpref . "words
12051232
WHERE WoLgID = " . $lid . " AND WoWordCount > 1
12061233
GROUP BY WoWordCount"
12071234
);
12081235
while ($record = mysqli_fetch_assoc($res)){
1209-
if($wl_max < $record['len']) {
1210-
$wl_max = $record['len'];
1236+
if ($wl_max < $record['word_count']) {
1237+
$wl_max = $record['word_count'];
12111238
}
1212-
$wl[] = $record['len'];
1213-
$mw_sql .= ' WHEN ' . $record['len'] .
1214-
' THEN @a' . (intval($record['len']) * 2 - 1);
1239+
$wl[] = $record['word_count'];
1240+
$mw_sql .= ' WHEN ' . $record['word_count'] .
1241+
' THEN @a' . (intval($record['word_count']) * 2 - 1);
12151242
}
12161243
mysqli_free_result($res);
12171244
$sql = '';
1218-
if(!empty($wl)) {//text has expressions
1245+
// Text has expressions
1246+
if (!empty($wl)) {
12191247
$sql = check_text_with_expressions($id, $lid, $wl, $wl_max, $mw_sql);
1220-
}//text has expressions end
1221-
if($id>0) {
1248+
}
1249+
if ($id > 0) {
12221250
update_default_values($id, $lid, $sql);
12231251
}
1224-
if($id==-1) {//check text
1252+
// Check text
1253+
if ($id == -1) {
12251254
check_text($sql, $rtlScript, $wl);
1226-
}//check text end
1255+
}
12271256
do_mysqli_query('TRUNCATE TABLE ' . $tbpref . 'temptextitems');
12281257
}
12291258

0 commit comments

Comments
 (0)