Skip to content

GitLab

I
ilmt-api-pan-shallowparser
Commit c4db6963 authored by priyank's avatar priyank
Browse files
Options

updated tokenizer

parent e5d334b1
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 33579 additions and 301 deletions
modules/ILMT-PAN-HIN-Tokenizer/lib/ILMT/PAN/HIN/Tokenizer.pm
View file @ c4db6963
......@@ -4,6 +4,13 @@ use warnings;
use Dir::Self;
use Data::Dumper;
use IPC::Run qw(run);
use List::UtilsBy qw(max_by);
use File::Temp qw/ tempfile /;
use File::Slurp qw( slurp );
my $cwd = __DIR__;
my %daemons = (
"tokenizer" => {
"path" => "ind-tokz",
......@@ -15,22 +22,18 @@ my %daemons = (
sub process {
my %args = @_;
utf8::encode($args{data});
my $sentences = call_daemon("tokenizer", $args{data});
open INFILE, '<', \$sentences or die $!;
my $result = "";
my $ctr = 0;
while (my $line = <INFILE>) {
$ctr ++;
$result .= "<Sentence id=\"$ctr\">\n";
my @words = split ' ', $line;
foreach my $index (0..$#words) {
$result .= $index + 1 . "\t$words[$index]\tunk\n";
}
$result .= "</Sentence>";
}
close INFILE;
utf8::decode($result);
return $result;
my ($fh2, $filename2) = tempfile("tokenizer_inputXXXX", DIR => "/tmp", SUFFIX => ".tmp");
print $fh2 $args{"data"};
close($fh2);
my $token_out;
run ["python", "$cwd/tokenize.py", $filename2], ">", \$token_out;
unlink $filename2 or die "Couldn't delete temp file! $filename2";
utf8::decode($token_out);
return $token_out;
}
sub run_daemons {
......
modules/ILMT-PAN-HIN-Tokenizer/lib/ILMT/PAN/HIN/Tokenizer.pm-old 0 → 100644
View file @ c4db6963
package ILMT::PAN::HIN::Tokenizer;
use strict;
use warnings;
use Dir::Self;
use Data::Dumper;
use IPC::Run qw(run);
use List::UtilsBy qw(max_by);
use File::Temp qw/ tempfile /;
use File::Slurp qw( slurp );
my $cwd = __DIR__;
my %daemons = (
"tokenizer" => {
"path" => "ind-tokz",
"args" => "--l pan --s --daemonize --port",
"port" => "21001"
}
);
sub process {
my %args = @_;
utf8::encode($args{data});
my $sentences = call_daemon("tokenizer", $args{data});
open INFILE, '<', \$sentences or die $!;
my $result = "";
my $ctr = 0;
while (my $line = <INFILE>) {
$ctr ++;
$result .= "<Sentence id=\"$ctr\">\n";
my @words = split ' ', $line;
foreach my $index (0..$#words) {
$result .= $index + 1 . "\t$words[$index]\tunk\n";
}
$result .= "</Sentence>";
}
print Dumper($result);
close INFILE;
utf8::decode($result);
return $result;
}
sub run_daemons {
my @daemon_names = @_;
foreach my $daemon_name (@daemon_names) {
my %daemon = %{$daemons{$daemon_name}};
my $cmd = "$daemon{path} $daemon{args} $daemon{port} &";
my $runfile = __DIR__ . "/run/${daemon_name}_$daemon{port}";
system("flock -e -w 0.01 $runfile -c '$cmd'") == 0
or warn "[" . __PACKAGE__ . "]: Port $daemon{port} maybe unavailable! $?\n";
}
}
sub call_daemon {
my ($daemon_name, $input) = @_;
my $port = $daemons{$daemon_name}{port};
my ($socket, $client_socket);
$socket = new IO::Socket::INET (
PeerHost => '127.0.0.1',
PeerPort => $port,
Proto => 'tcp',
) or die "ERROR in Socket Creation : $!\n";
$socket->send("$input\n");
my $result = "";
while (my $line = $socket->getline) {
$result .= $line;
}
$socket->close();
return $result;
}
run_daemons(("tokenizer"));
1;
modules/ILMT-PAN-HIN-Tokenizer/lib/ILMT/PAN/HIN/tokenize.py 0 → 100644
View file @ c4db6963
import os, sys, codecs
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Created by
@author: priyank
'''
def tokenizer(text, ind):
"""Tokenize the text only on space."""
tokens = text.split()
tokens_ssf = [str(index + 1) + '\t' + token + '\tunk' for index, token in enumerate(tokens)]
tokens_ssf_with_sentence = ['<Sentence id="'+str(ind+1)+'">'] + tokens_ssf + ['</Sentence>']
return '\n'.join(tokens_ssf_with_sentence)
f = codecs.open(sys.argv[1], "rb", "utf-8")
lines = f.readlines()
f.close()
finalOutput = ""
ii = 0
for line in lines:
line = line.strip()
if line:
finalOutput = finalOutput + tokenizer(line, (ii)) + "\n"
ii = ii + 1
print (finalOutput.encode('utf-8'))
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF.pm-old 0 → 100644
View file @ c4db6963
This diff is collapsed.
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/ChangeLog 0 → 100644
View file @ c4db6963
14-Mar-2011 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.7
- In wx2utf the words start with ^ or ^@ just leave as it is.
- In case of tamil wx2utf 0BB6 (S) mapped to 0BB7 (R)
- M map is changed m+halant (as per Dr. Ramnan feedback)
7-Mar-2011 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.6
- iscii2unicode_tel function some mapping commented like ऴ etc.
28-Feb-2011 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.5
- fix ऩ issue from wx2utf in telugu.
- case added in tests directory (sampale-cases-tel-wx.txt)
- tamil normailze issue two-part dependent vowel signs solved.
21-Feb-2011 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.4
- fix punc issue in ssf input.
- fix ~ issue in input and output file path.
06-Apr-2010 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.3
- incorporate the output option from the command line option.
- create convertor_indic_lib.pl for convertor library.
- create sample-convertor-call.pl for how to call convertor library.
28-Feb-2011 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.5
- fix ऩ issue from wx2utf in telugu.
- case added in tests directory (sampale-cases-tel-wx.txt)
- tamil normailze issue two-part dependent vowel signs solved.
21-Feb-2011 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.4
- fix punc issue in ssf input.
- fix ~ issue in input and output file path.
06-Apr-2010 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.3
- incorporate the output option from the command line option.
- create convertor_indic_lib.pl for convertor library.
- create sample-convertor-call.pl for how to call convertor library.
18-Mar-2010 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.2
- wx2utf.pl and utf2wx.pl files function are moved in lib/IndicCC.pl.
- incorporate the small code in wx2utf function in lib/IndicCC.pl.
- to escap the other language string as it is.
18-Mar-2010 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.1
- tested 1050 uniq words. input file in tests/tam/uniq-words-tam-aukbc-[wx|utf].txt
- isuue related wx2utf are fixed by incorporating the variou case in wx2iscii sub. in lib/IndicCC.pl
- issue reagrding utf2wx are fixed by incorporating 3-lines in iscii2wx sub. in lib/IndicCC.pl
- "\x{D0}"=> rY, "\x{C7}"=>nY, "\x{D3}"=>lYY
- added the support of tamil language.
15-Mar-2010 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.4.0
- test wx2utf-tel, utf2wx-tel and fix the various cases like lYa,lYi, lY[MHz] etc.
- resolve full stop issue from wx2utf direction for hindi.
- we have compare the changed IndicCC.pl to old one and undo the changes
that Harika Indu did.
08-May-2010 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.3.0
- text conversion incorporated.
- input option incorporated in command line argument parsing.
- tel ssf test data incorporated.
03-03-2010 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.2.0
- changed as per requirement of ILMT project Sampark.
- this version is currently working on SSF format only.
- test on hindi data ONLY.
08-02-10 Rashid Ahmad <rashid101b@gmail.com>
* Version 1.1.0
- This version is modified of the base version.
- created the sample pl file for test convertor.
- Base version is developed in perl by Pawan Kumar, Rashid Ahmad and Avinesh.
10-Aug-09 Pawan Kumar <hawahawai@gmail.com>
* Version 1.0.0
- Base Version.
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/README 0 → 100644
View file @ c4db6963
convertor-indic :
convertor is used for converting the input file into utf format or in wx
format.The fields which are converting is TKN,lex,vib,name and head in SSF.
If the input file is in wx format the program converts only the words which
are in wx format irrespective of them starting with or without "@" symbol
and if the inputfile is in utf format the program converts only the words
which are in the source language mentioned by the user irrespective of them
starting with or without "@" symbol. The converter will not touch any third
language( other than source language) Unicode character .( i.e) while
processing. The input file must be in SSF or TEXT format.
How to use ??
1) convertor-indic (wx2utf or utf2wx)
perl convertor_indic.pl --format=[ssf|text] --lang=[hin|tel|..] --src_encoding=[utf|wx] \
--tgt_encoding=[wx|utf] --input=<input-file>
e.g.
a) UTF to WX for Hindi (SSF format)
perl convertor_indic.pl -f=ssf -l=hin -s=utf -t=wx -i=tests/hin/ssf/test_case_3_utf.in
output will be printed to STDOUT
b) WX to UTF fo Hindi (SSF format)
perl convertor_indic.pl -f=ssf -l=hin -s=wx -t=utf -i=tests/hin/ssf/test_case_3_wx.in
output will be printed to STDOUT
c) UTF to WX for Hindi (TEXT format)
perl convertor_indic.pl -f=text -l=hin -s=utf -t=wx -i=tests/hin/text/sample_story_utf.in
d) WX to UTF fo Hindi (TEXT format)
perl convertor_indic.pl -f=text -l=hin -s=wx -t=utf -i=tests/hin/text/sample_story_wx.in
output will be printed to STDOUT
e) more information check
perl convertor_indic.pl --help
Directory Structure:
convertor-indic
|
|---lib (source code of the convertor library)
|
|---ssfapi (SSF API's)
|
|---tests (contains the referenece input and output)
|
|---doc (documentaion)
|
|---extra-files (some backup/extra files)
|
|---convertor_indic.pl (main file)
|
|---wx2utf.pl (conversion wx2utf file)
|
|---utf2wx.pl (conversion utf2wx file)
|
|---README (How to run/use the module)
|
|---ChangeLog (version information)
Contact :
Rashid Ahmad
Expert Software Ltd.
rashid101b@gmail.com
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/in 0 → 100644
View file @ c4db6963
<Sentence id="2">
1 हजारिका unk
2 का unk
3 गत unk
4 पांच unk
5 नवंबर unk
6 को unk
7 कई unk
8 अंगों unk
9 के unk
10 काम unk
11 करना unk
12 बंद unk
13 करने unk
14 के unk
15 कारण unk
16 एक unk
17 अस्पताल unk
18 में unk
19 निधन unk
20 हो unk
21 गया unk
22 था unk
23 . unk
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/case-caret_at-wx.in 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 (( NP <fs af='kevalaM,n,,sg,,d,0,0' head="kevalaM" name=1>
1.1 kevalaM NN <fs af='kevalaM,n,,sg,,d,0,0' name="kevalaM">
))
2 (( RBP <fs af='hAyigA,avy,,,,,0,0_avy' head="hAyigA" name=2 poslcat="NM">
2.1 hAyigA RB <fs af='hAyigA,avy,,,,,0,0_avy' poslcat="NM" name="hAyigA">
))
3 (( NP <fs af='jIviwaM,n,,sg,,,nu,nu' head="jIviwAnni" vpos="vib1" name=3>
3.1 jIviwAnni NN <fs af='jIviwaM,n,,sg,,,nu,nu' name="jIviwAnni">
))
4 (( VGNF <fs af='gadapu,v,any,any,any,,adaM,adaM' head="gadapadaM" name=4>
4.1 @kalYYi VM <fs af='^@kalYYi,v,any,any,any,,adaM,adaM' name="gadapadaM">
))
5 (( VGNF <fs af='vaccu,v,any,any,any,,we,we' head="vaswe" name=5>
5.1 ^weVri VM <fs af='^@weVri,v,any,any,any,,we,we' name="vaswe">
))
6 (( VGNF <fs af='cAlu,v,,sg,2,,AjFArWa,AjFArWa' head="cAlu" name=6>
6.1 ^@cAlu VM <fs af='@cAlu,v,,sg,2,,@AjFArWa,AjFArWa' name="cAlu">
))
7 (( VGNF <fs af='anu,v,fm,pl,3,,wunn,wunn' head="aMtunnAru" name=7>
7.1 aMtunnAru VM <fs af='anu,v,fm,pl,3,,wunn,wunn' name="aMtunnAru">
))
8 (( NP <fs af='mAnasika,n,,sg,,d,0,0' head="mAnasika" name=8>
8.1 manYowawwuva NN <fs af='^@manYowawwuva,n,,sg,,d,^@mA,0' name="mAnasika">
))
9 (( NP <fs af='vExyanipuNudu,n,,pl,,d,0,0' head="vExyanipuNulu" name=9>
9.1 vallunar NN <fs af='^@vallunar,n,,pl,,d,^kA,0' name="vExyanipuNulu">
9.2 . SYM <fs af='&dot;,punc,,,,,,'>
))
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/case-caret_at-wx2utf-tel.out 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 (( NP <fs af='కేవలం,n,,sg,,d,0,0' head="కేవలం" name='1'>
1.1 కేవలం NN <fs af='కేవలం,n,,sg,,d,0,0' name="కేవలం">
))
2 (( RBP <fs af='హాయిగా,avy,,,,,0,0_avy' head="హాయిగా" name='2' poslcat="NM">
2.1 హాయిగా RB <fs af='హాయిగా,avy,,,,,0,0_avy' name="హాయిగా" poslcat="NM">
))
3 (( NP <fs af='జీవితం,n,,sg,,,ను,nu' vpos="vib1" head="జీవితాన్ని" name='3'>
3.1 జీవితాన్ని NN <fs af='జీవితం,n,,sg,,,ను,nu' name="జీవితాన్ని">
))
4 (( VGNF <fs af='గడపు,v,any,any,any,,అడం,adaM' head="గడపడం" name='4'>
4.1 @kalYYi VM <fs af='^@kalYYi,v,any,any,any,,అడం,adaM' name="గడపడం">
))
5 (( VGNF <fs af='వచ్చు,v,any,any,any,,తే,we' head="వస్తే" name='5'>
5.1 ^తెరి VM <fs af='^@weVri,v,any,any,any,,తే,we' name="వస్తే">
))
6 (( VGNF <fs af='చాలు,v,,sg,2,,ఆజ్ఞార్థ,AjFArWa' head="చాలు" name='6'>
6.1 ^@cAlu VM <fs af='@cAlu,v,,sg,2,,@AjFArWa,AjFArWa' name="చాలు">
))
7 (( VGNF <fs af='అను,v,fm,pl,3,,తున్న్,wunn' head="అంటున్నారు" name='7'>
7.1 అంటున్నారు VM <fs af='అను,v,fm,pl,3,,తున్న్,wunn' name="అంటున్నారు">
))
8 (( NP <fs af='మానసిక,n,,sg,,d,0,0' head="మానసిక" name='8'>
8.1 మోతత్తువ NN <fs af='^@manYowawwuva,n,,sg,,d,^@mA,0' name="మానసిక">
))
9 (( NP <fs af='వైద్యనిపుణుడు,n,,pl,,d,0,0' head="వైద్యనిపుణులు" name='9'>
9.1 వల్లునర్ NN <fs af='^@vallunar,n,,pl,,d,^కా,0' name="వైద్యనిపుణులు">
9.2 . SYM <fs af='&dot;,punc,,,,,,'>
))
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/sample_hin_utf.in 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 (( NP <fs af='राशिद,pn,any,sg,1,o,टेम,ne' head='मैंने'>
1.1 राशि PRP <fs af='राशिद,pn,any,sg,1,o,ने,ne' name="मैंने">|<fs af='मैं,pn,any,sg,1,o,मे,ne' name="मैंने">
))
2 (( VGNF <fs af='दौड़,v,m,pl,any,,ता,wA' head='दौड़ते'>
2.1 दौड़ते VM <fs af='जेंङो,v,m,pl,any,,ता,wA' name='दौड़ते'>
2.2 हुए VAUX <fs af='हो,v,m,pl,any,,या,yA' name='हुए' poslcat='NM'>
))
3 (( NP <fs af='शेर,n,m,sg,3,o,0,0' head='शेर'>
3.1 शेर NN <fs af='शेर,n,m,sg,3,o,0,0' name='शेर'>
3.2 को PSP <fs af='को,psp,,,,,,' name='को'>
))
4 (( VGF <fs af='राशिद,v,m,sg,any,,या,yA' head='देखा'>
4.1 राशि VM <fs af='देख,v,m,sg,any,,या,yA' name='देखा'>
4.2 । SYM <fs af='।,punc,,,,,,' name='।'>
))
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/sample_hin_utf.out 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 (( NP <fs af='rASixa,pn,any,sg,1,o,tema,ne' head='mEMne'>
1.1 rASi PRP <fs af='rASixa,pn,any,sg,1,o,ne,ne' name="mEMne">|<fs af='mEM,pn,any,sg,1,o,me,ne' name="mEMne">
))
2 (( VGNF <fs af='xOdZa,v,m,pl,any,,wA,wA' head='xOdZawe'>
2.1 xOdZawe VM <fs af='jeMfo,v,m,pl,any,,wA,wA' name='xOdZawe'>
2.2 hue VAUX <fs af='ho,v,m,pl,any,,yA,yA' name='hue' poslcat='NM'>
))
3 (( NP <fs af='Sera,n,m,sg,3,o,0,0' head='Sera'>
3.1 Sera NN <fs af='Sera,n,m,sg,3,o,0,0' name='Sera'>
3.2 ko PSP <fs af='ko,psp,,,,,,' name='ko'>
))
4 (( VGF <fs af='rASixa,v,m,sg,any,,yA,yA' head='xeKA'>
4.1 rASi VM <fs af='xeKa,v,m,sg,any,,yA,yA' name='xeKA'>
4.2 . SYM <fs af='.,punc,,,,,,' name='.'>
))
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/sample_hin_wx.in 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 (( NP <fs af='rASixa,pn,any,sg,1,o,tema,ne' head='mEMne'>
1.1 rASi PRP <fs af='rASixa,pn,any,sg,1,o,ne,ne' name="mEMne">|<fs af='mEM,pn,any,sg,1,o,me,ne' name="mEMne">
))
2 (( VGNF <fs af='xOdZa,v,m,pl,any,,wA,wA' head='xOdZawe'>
2.1 xOdZawe VM <fs af='jeMfo,v,m,pl,any,,wA,wA' name='xOdZawe'>
2.2 hue VAUX <fs af='ho,v,m,pl,any,,yA,yA' name='hue' poslcat='NM'>
))
3 (( NP <fs af='Sera,n,m,sg,3,o,0,0' head='Sera'>
3.1 Sera NN <fs af='Sera,n,m,sg,3,o,0,0' name='Sera'>
3.2 ko PSP <fs af='ko,psp,,,,,,' name='ko'>
))
4 (( VGF <fs af='rASixa,v,m,sg,any,,yA,yA' head='xeKA'>
4.1 rASi VM <fs af='xeKa,v,m,sg,any,,yA,yA' name='xeKA'>
4.2 . SYM <fs af='.,punc,,,,,,' name='.'>
))
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/sample_hin_wx.out 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 (( NP <fs af='राशिद,pn,any,sg,1,o,टेम,ne' head='मैंने'>
1.1 राशि PRP <fs af='राशिद,pn,any,sg,1,o,ने,ne' name="मैंने">|<fs af='मैं,pn,any,sg,1,o,मे,ne' name="मैंने">
))
2 (( VGNF <fs af='दौड़,v,m,pl,any,,ता,wA' head='दौड़ते'>
2.1 दौड़ते VM <fs af='जेंङो,v,m,pl,any,,ता,wA' name='दौड़ते'>
2.2 हुए VAUX <fs af='हो,v,m,pl,any,,या,yA' name='हुए' poslcat='NM'>
))
3 (( NP <fs af='शेर,n,m,sg,3,o,0,0' head='शेर'>
3.1 शेर NN <fs af='शेर,n,m,sg,3,o,0,0' name='शेर'>
3.2 को PSP <fs af='को,psp,,,,,,' name='को'>
))
4 (( VGF <fs af='राशिद,v,m,sg,any,,या,yA' head='देखा'>
4.1 राशि VM <fs af='देख,v,m,sg,any,,या,yA' name='देखा'>
4.2 । SYM <fs af='।,punc,,,,,,' name='।'>
))
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/test-ssf-wx.in 0 → 100644
View file @ c4db6963
1 rAma
2 ਰਹ੍ਯਿਣਾ
3 రాశిద్
4 jAwA
5 hE.
6 60vAM
7 राशिद
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/test-ssf-wx.out 0 → 100644
View file @ c4db6963
1 राम
2 ਰਹ੍ਯਿਣਾ
3 రాశిద్
4 जाता
5 है.
6 60वां
7 राशिद
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/test_case_1_utf.in 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 हरिद्वार unk
2 में unk
3 माँ unk
4 गंगा unk
5 एक unk
6 सिरे unk
7 से unk
8 दूसरे unk
9 सिरे unk
10 तक unk
11 बहती unk
12 हैं unk
13 @ZCOMMA unk
14 लेकिन unk
15 जो unk
16 पुण्य unk
17 हर unk
18 की unk
19 पौड़ी unk
20 में unk
21 स्थित unk
22 ' unk
23 ब्रह्मकुंड unk
24 ' unk
25 में unk
26 स्नान unk
27 से unk
28 मिलता unk
29 है unk
30 @ZCOMMA unk
31 वह unk
32 कहीं unk
33 नहीं unk
34 मिलता unk
35 । unk
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/test_case_1_utf.out 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 harixvAra unk
2 meM unk
3 mAz unk
4 gaMgA unk
5 eka unk
6 sire unk
7 se unk
8 xUsare unk
9 sire unk
10 waka unk
11 bahawI unk
12 hEM unk
13 @ZCOMMA unk
14 lekina unk
15 jo unk
16 puNya unk
17 hara unk
18 kI unk
19 pOdZI unk
20 meM unk
21 sWiwa unk
22 ' unk
23 brahmakuMda unk
24 ' unk
25 meM unk
26 snAna unk
27 se unk
28 milawA unk
29 hE unk
30 @ZCOMMA unk
31 vaha unk
32 kahIM unk
33 nahIM unk
34 milawA unk
35 . unk
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/ssf/test_case_1_wx.in 0 → 100644
View file @ c4db6963
<Sentence id="1">
1 harixvAra unk
2 meM unk
3 mAz unk
4 gaMgA unk
5 eka unk
6 sire unk
7 se unk
8 xUsare unk
9 sire unk
10 waka unk
11 bahawI unk
12 hEM unk
13 @ZCOMMA unk
14 lekina unk
15 jo unk
16 puNya unk
17 hara unk
18 kI unk
19 pOdZI unk
20 meM unk
21 sWiwa unk
22 ' unk
23 brahmakuMda unk
24 ' unk
25 meM unk
26 snAna unk
27 se unk
28 milawA unk
29 hE unk
30 @ZCOMMA unk
31 vaha unk
32 kahIM unk
33 nahIM unk
34 milawA unk
35 . unk
</Sentence>
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/hin/text/sample_story_utf.in 0 → 100644
View file @ c4db6963
एक गाय भी थी । पिछले साल पता नहीं कौन सी बीमारी लगी और वह चल बसी । गाय का मरना माई को बहुत खला । अब भी जब कभी उसकी याद आती माई का झुर्रियों वाला चेहरा लटक जाता है । गाय की मरते वक्त भी छटपटाहट उसकी आँखों के सामने घूम जाती है । खेत बँटाई पर है । कुछ न कुछ मिल ही जाता है । पेट काट कर माई ने जिस तरह उसे पाला है, वह अच्छी तरह समझता है । उसी के लिए माई देहात नहीं छोड़ रही, इसे भी वह समझता है ।
भाई अकेलापन न महसूस करने पाए इसलिए बच्चों को महीने दो महीने के लिए गाँव छोड़ आता है । खुद भी चला जाता है । ईद तो उसने हमेशा गाँव में ही मनाई । परदेश में कौन अपना होता है । कौन किसकी परवाह करता है । सोचते सोचते वह बिस्तर पर लेट गया । मोटी वाली नर्स इस बीच आई । दवा खिला कर चली गई । अगल बगल के मरीज़ों को करीम जानता है । एक तरफ महाराजदीन हैं तो दूसरी तरफ रहमत मियाँ । दोनों उदास रहते हैं । चेहरा पीला हो गया है । दाढ़ी बाल इस कदर बढ़ आये हैं कि सूरत पहचानी नहीं जाती । इनसे मिलने कोई नहीं आता ।
महाराजदीन कहता है कि उसके भगवान हैं जो रोज शाम सवेरे उनसे मिलने आते हैं । रहमत कुछ नहीं कहता । कहते कहते रुक जाता है । ये दोनों मिलने वालों को हसरत भरी निगाहों से देखते हैं । यहाँ अपना कोई नहीं है । बाल बच्चे देहात में रहते हैं । उन्हें खबर भी नहीं होगी । खबर देने से कोई फ़ायदा नहीं घर वालों को परेशानी में डालना उन्हें अच्छा न लगा होगा ।
modules/ILMT-PAN-HIN-WX2UTF/lib/ILMT/PAN/HIN/WX2UTF/convertor-indic-1.4.7/tests/pan/test-1-utf.txt 0 → 100644
View file @ c4db6963
ਸਾਹਿੱਤਕ ਦੌਰ ਵਿਚ ਪਹਿਲਾਂ ਰਾਬਿੰਦਰ ਸਿੰਘ ਅਟਵਾਲ ਨੇ “ਬਦਲਾ”, ਮੁਹਿੰਦਰ ਸਿੰਘ ਘੱਗ ਨੇ “ਨਸ਼ਾ ਆਪੋ ਆਪਣਾ” ਤਤਿੰਦਰ ਕੌਰ ਨੇ “ਪੱਪੀ ਲਵ” ਮਨਜੀਤ ਸ਼ੇਖੋਂ ਨੇ “ ਨਿਪੱਤਰਾ ਰੁਖ” ਕਹਾਣੀਆਂ ਪੜ੍ਹੀਆਂ।
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment