It's easy to write a text pre-processing template. But to process the text in the future, we will not repeat it. As the number of texts processed increases, will be updated slowly ..
1 #! /Usr/bin/enc python
2 #-*-coding: UTF-8 -*-
3 #
4 #
5 # Copyrigtht (c)
6 # Laiseek co1_2012
7 # All rights reserved.
8 #
9 # Finename: nearsyn. py
10 # Function: preprocessing Tool
11 #
12 # Current version: 1.0
13 # author: Chen Yu
14 # Date: 05/07/2012
15 #
16
17 class NearSyn:
18 def init (self ):
19 self. syn = []
20 self. det = []
21 # load data
22 def load (self, filename, detfile ):
23 sock = open (filename, 'R ')
24 self. syn = sock. read (). split ('\ n ')
25 sock. close ()
26 for I in range (len (self. syn )):
27 self. syn [I] = self. syn [I]. split ('')
28 self. det = []
29 sock = open (detfile, 'R ')
30 self. det = sock. read (). split ('\ n ')
31 sock. close ()
32
33 # Remove the empty list in the read Element
34 def empty (self ):
35 for I in range (len (self. syn )):
36 # reduce the number of empty elements after Removal
37 flag = 0
38 for j in range (len (self. syn [I]):
39 if self. syn [I] [j-flag] = "" or len (self. syn [I] [j-flag]) = 0:
40 self. syn [I]. remove (self. syn [I] [j-flag])
41 flag + = 1
42
43 # Remove D from
44 def dete (self ):
45 for I in range (len (self. syn )):
46 self. det [I] = self. det [I]. split ('')
47 for I in range (len (self. syn )):
48 for j in range (len (self. det [I]):
49 if self. det [I] [j]:
50 self. syn [I]. remove (self. det [I] [j])
51
52 # output www.2cto.com
53 def output (self ):
54 sock = open ('nearsync', 'w ')
55 for I in range (len (self. syn )):
56 # Remove a single word
57 if len (self. syn [I])> 1:
58 for j in range (len (self. syn [I]):
59 sock. write (self. syn [I] [j] + '')
60 sock. write ("\ n ")
61 sock. close ()
62
63 if _ name _ = '_ main __':
64 t = NearSyn ()
65 t. load ('A', 'D ')
66 t. empty ()
67 t. dete ()
68 t. output ()
From sunrise