PYTHON3+PYQT5 implementation of a page indexer application that supports multithreading

This article is mainly for you to introduce the PYTHON3+PYQT5 implementation support multi-threaded page indexer application, with a certain reference value, interested in small partners can refer to

This paper implements a 19-chapter page Indexer Application example of the Python Qt GUI fast programming through PYTHON3+PYQT5.

/home/yrd/eric_workspace/chap19/walker_ans.py

#!/usr/bin/env python3import codecsimport html.entitiesimport reimport sysfrom pyqt5.qtcore Import (QMutex, QThread, PYQTSIGNAL,QT) class Walker (Qthread): finished = pyqtsignal (bool,int) indexed = pyqtsignal (str,int) Common_words_ THRESHOLD = Min_word_len = 3 Max_word_len = Invalid_first_or_last = Frozenset ("0123456789_") Striphtml_re = Re.comp Ile (R "<[^>]*?>", re. Ignorecase|re. MULTILINE) Entity_re = Re.compile (r "& (\w+?); | (\d+?); ") Split_re = Re.compile (r "\w+", RE. Ignorecase|re. MULTILINE) def __init__ (self, index, lock, files, Filenamesforwords, Commonwords, Parent=none): Super (Walker, self). _ _init__ (parent) Self.index = Index Self.lock = Lock Self.files = files Self.filenamesforwords = filenamesforwords sel F.commonwords = Commonwords self.stopped = False Self.mutex = Qmutex () self.completed = False def Stop (self): try:s Elf.mutex.lock () self.stopped = True Finally:self.mutex.unlock () def isStopped (self): Try:self.mutex.lock () re Turn Self.stoPped Finally:self.mutex.unlock () def run (self): Self.processfiles () self.stop () Self.finished.emit (self.completed,s ELF.INDEX) def processfiles (self): def unichrfromentity (match): Text = Match.group (match.lastindex) if Text.isdigit () : Return chr (int (text)) U = html.entities.name2codepoint.get (text) return Chr (U) if u is not None else "" for Fnam E in Self.files:if self.isstopped (): Return words = set () fh = None TRY:FH = Codecs.open (fname, "R", "UTF8    "," ignore ") Text = Fh.read () except EnvironmentError as E:sys.stderr.write (" Error: {0}\n ". Format (e)) continue Finally:if FH is isn't None:fh.close () if self.isstopped (): Return text = self. Striphtml_re.sub ("", text) Text = self. Entity_re.sub (unichrfromentity, text) Text = Text.lower () for word in self. Split_re.split (text): if (self. Min_word_len <= LEN (WORD) <= self. Max_word_len and word[0] not in self. Invalid_first_or_last and word[-1] not in self. InvaLid_first_or_last): Try:self.lock.lockForRead () new = Word not in self.commonwords FINALLY:SELF.L Ock.unlock () if New:words.add (word) if self.isstopped (): Return for word in Words:try:self.lock. Lockforwrite () files = Self.filenamesforwords[word] If len (files) > self. Common_words_threshold:del Self.filenamesforwords[word] Self.commonWords.add (word) else:files.add (str    (fname)) Finally:self.lock.unlock () self.indexed.emit (fname,self.index) self.completed = True/home/yrd/eric_workspace/chap1 9/pageindexer_ans.pyw#!/usr/bin/env python3import collectionsimport osimport sysfrom PyQt5.QtCore import (QDir, Qreadwritelock, QMUTEX,QT) from pyqt5.qtwidgets import (Qapplication, Qdialog, Qfiledialog, Qframe, Qhboxlayout, QLC Dnumber, Qlabel, Qlineedit, Qlistwidget, Qpushbutton, qvboxlayout) import Walker_ans as Walkerdef isAlive (qobj): Imp Ort sip Try:sip.unwrapinstance (qobj) except RuntimeError: Return False return Trueclass Form (qdialog): Def __init__ (self, Parent=none): Super (Form, self). __init__ (parent) self.m Utex = Qmutex () self.filecount = 0 Self.filenamesforwords = collections.defaultdict (set) Self.commonwords = set () Self . Lock = Qreadwritelock () Self.path = Qdir.homepath () Pathlabel = Qlabel ("Indexing path:") Self.pathlabel = Qlabel () se Lf.pathLabel.setFrameStyle (qframe.styledpanel| Qframe.sunken) Self.pathbutton = Qpushbutton ("Set &path ...") Self.pathButton.setAutoDefault (False) Findlabel = QLa Bel ("&find Word:") Self.findedit = Qlineedit () findlabel.setbuddy (self.findedit) Commonwordslabel = QLabel ("&C  Ommon words: ") Self.commonwordslistwidget = Qlistwidget () commonwordslabel.setbuddy (Self.commonwordslistwidget) Fileslabel = Qlabel ("Files containing the &word:") Self.fileslistwidget = Qlistwidget () Fileslabel.setbuddy (self.fi Leslistwidget) Filesindexedlabel = Qlabel ("Files indexed") SELF.FILESINDEXEDLCD = Qlcdnumber () self.filesinDexedlcd.setsegmentstyle (qlcdnumber.flat) Wordsindexedlabel = Qlabel ("Words indexed") SELF.WORDSINDEXEDLCD =  Qlcdnumber () Self.wordsIndexedLCD.setSegmentStyle (qlcdnumber.flat) Commonwordslcdlabel = Qlabel ("Common words") SELF.COMMONWORDSLCD = Qlcdnumber () self.commonWordsLCD.setSegmentStyle (qlcdnumber.flat) Self.statuslabel = Qlabel (" Click the ' Set Path ' "" button to start Indexing ") Self.statusLabel.setFrameStyle (qframe.styledpanel| Qframe.sunken) Toplayout = Qhboxlayout () toplayout.addwidget (Pathlabel) Toplayout.addwidget (Self.pathLabel, 1) topLay Out.addwidget (Self.pathbutton) toplayout.addwidget (Findlabel) Toplayout.addwidget (Self.findEdit, 1) leftLayout = Qvboxlayout () leftlayout.addwidget (Fileslabel) leftlayout.addwidget (self.fileslistwidget) rightLayout = QVBoxLayout () Rightlayout.addwidget (Commonwordslabel) rightlayout.addwidget (self.commonwordslistwidget) middleLayout = Qhboxlayout () middlelayout.addlayout (leftlayout, 1) middlelayout.addlayout (rightlayout) Bottomlayout = Qhboxlayout () bottomlayout.addwidget (Filesindexedlabel) Bottomlayout.addwidget (  SELF.FILESINDEXEDLCD) Bottomlayout.addwidget (Wordsindexedlabel) bottomlayout.addwidget (Self.wordsIndexedLCD) Bottomlayout.addwidget (Commonwordslcdlabel) bottomlayout.addwidget (SELF.COMMONWORDSLCD) BottomLayout.addStretch ( ) layout = Qvboxlayout () layout.addlayout (toplayout) layout.addlayout (middlelayout) layout.addlayout (bottomLayout) LA Yout.addwidget (Self.statuslabel) self.setlayout (layout) self.walkers = [] self.completed = [] self.pathButton.clicked. Connect (Self.setpath) self.findEdit.returnPressed.connect (self.find) self.setwindowtitle ("Page Indexer") def Stopwalkers (self): for Walker in Self.walkers:if IsAlive (Walker) and Walker.isrunning (): Walker.stop () for Walker  In Self.walkers:if isAlive (Walker) and Walker.isrunning (): walker.wait () self.walkers = [] self.completed = [] def SetPath (self): Self.stopwalkers () self.pathButton.setEnabled (FalsePath = Qfiledialog.getexistingdirectory (self, ' Choose a path to Index ', Self.path) if not Path:self.statusLabel . SetText ("Click the ' Set Path '" "button to start Indexing") self.pathButton.setEnabled (True) return Self.st Atuslabel.settext ("Scanning directories ...") qapplication.processevents () # Needed for Windows Self.path = Qdir.tonativ  Eseparators (Path) Self.findEdit.setFocus () Self.pathLabel.setText (Self.path) self.statusLabel.clear ()  Self.filesListWidget.clear () self.filecount = 0 Self.filenamesforwords = collections.defaultdict (set)   Self.commonwords = set () Nofilesfound = True files = [] index = 0 for root, dirs, fnames in Os.walk (str (self.path)): For name in [name of name in Fnames if Name.endswith ((". htm", ". html")]: Files.append (Os.path.join (root, name)  If len (files) = = 1000:self.processfiles (index, files[:]) files = [] Index + = 1 Nofilesfound = False If Files:self.processFiles (Index, files[:]) NOFILESFOund = False If nofilesfound:self.finishedIndexing () self.statusLabel.setText ("No HTML files found in the given Path ") def processfiles (self, Index, files): thread = Walker. Walker (index, Self.lock, files, self.filenamesforwords, self.commonwords, self) thread.indexed[str,int].connect (self  . Indexed) Thread.finished[bool,int].connect (self.finished) thread.finished.connect (Thread.deletelater) Self.walkers.append (thread) self.completed.append (False) Thread.Start () thread.wait () # Needed for Windows def find ( Self): Word = str (self.findEdit.text ()) if not Word:try:self.mutex.lock () self.statusLabel.setText ("Enter a wo Rd to find in Files ") Finally:self.mutex.unlock () return Try:self.mutex.lock () self.statusLabel.clear () se  Lf.filesListWidget.clear () finally:self.mutex.unlock () word = Word.lower () if "" in Word:word = Word.split () [0] Try:self.lock.lockForRead () found = Word in self.commonwords finally:self.lock.unlock () if Found:try:self.mutex.lock () self.statusLabel.setText ("Common words like ' {0} '" "is not indexed". Format (w ORD)) Finally:self.mutex.unlock () return try:self.lock.lockForRead () files = Self.filenamesForWords.get (word , set ()). Copy () Finally:self.lock.unlock () if not Files:try:self.mutex.lock () self.statusLabel.setText ("No Indexed file contains "" The word ' {0} ' ". Format (Word)) Finally:self.mutex.unlock () return files = [Qdir.tona Tiveseparators (name) for name in sorted (files, key=str.lower)] Try:self.mutex.lock () self.filesListWidget.addIte MS (files) self.statusLabel.setText ("{0} indexed files contain the word ' {1} '". Format (len (files), word)) final Ly:self.mutex.unlock () def indexed (self, fname, index): Try:self.mutex.lock () Self.statusLabel.setText (fname) s Elf.filecount + = 1 count = Self.filecount finally:self.mutex.unlock () if count% = = 0:TRY:SELF.LOCK.LOCKF Orread () INDEXEDWORDCOunt = Len (self.filenamesforwords) Commonwordcount = Len (self.commonwords) Finally:self.lock.unlock () try:s Elf.mutex.lock () Self.filesIndexedLCD.display (count) self.wordsIndexedLCD.display (Indexedwordcount) self.commonwo Rdslcd.display (Commonwordcount) finally:self.mutex.unlock () elif count% 101 = = 0:try:self.lock.lockforread ( ) words = Self.commonWords.copy () finally:self.lock.unlock () Try:self.mutex.lock () SELF.COMMONWORDSLISTW Idget.clear () Self.commonWordsListWidget.addItems (sorted (words)) Finally:self.mutex.unlock () def finished (self, c ompleted, index): done = False if self.walkers:self.completed[index] = True if All (self.completed): Try:sel    F.mutex.lock () Self.statusLabel.setText ("finished") Done = True Finally:self.mutex.unlock () Else:try:   Self.mutex.lock () Self.statusLabel.setText ("finished") Done = True Finally:self.mutex.unlock () If-Done: Self.finishedindexing ()Def reject (self): If isn't all (self.completed): Self.stopwalkers () self.finishedindexing () else:self.accept () def c Loseevent (self, Event=none): Self.stopwalkers () def finishedindexing (self): Self.filesIndexedLCD.display ( Self.filecount) Self.wordsIndexedLCD.display (len (self.filenamesforwords)) Self.commonWordsLCD.display (Len ( self.commonwords)) self.pathButton.setEnabled (True) qapplication.processevents () # Needed for Windowsapp = Qapplication (sys.argv) Form = form () form.show () app.exec_ ()

Operation Result: