Monitor directory file changes with Golang

This is a creation in Article, where the information may have evolved or changed.

Want to write a program, monitor the directory and file changes, the original directory is very large, so feel to use Goroutine to each directory to derive a goroutine process, but the program at run time to find that open directory is very many, so that the system error, we first look at this failed program, There is no problem with a small directory.

Main.gopackage Mainimport (//"FMT" "Io/ioutil" "Log" "OS" "Path/filepath"//"RegExp" "Runtime" "Strings" "github.com/ Fsnotify/fsnotify ") type Fm struct {Basedir stringwatcher *fsnotify. Watcherwdone Chan Booldirchan Chan string}func (FM *fm) Init (Basedir string) {FM. Basedir = filepath. Fromslash (Basedir) var err errorfm. Watcher, err = fsnotify. Newwatcher () if err! = Nil {log. Fatal ("Create Watcher Error:", err)}fm. Wdone = Make (chan bool) go func () {for {select {case event: = <-fm. Watcher.events://log. Println ("Event:", event) Fm.process_event (event) Case ERR: = <-fm. Watcher.Errors:log. PRINTLN ("Watcher Error:", Err)}}} () FM. Dirchan = Make (Chan string,}FUNC (FM *FM) walkdir (path string) {//log. Println ("Basedir:", PATH) FM. Dirchan <-Pathdir, err: = Ioutil. ReadDir (PATH) if err! = Nil {log. Fatal ("Opendir:", Err)}for _, Fi: = Range dir {fpath: = filepath. Fromslash (path + "/" + fi.) Name ()) if Fi. Isdir () {if strings. Hasprefix (FI. Name (), ".") {continue}if strings. Hasprefix (FI. Name (), "..") {ContiNue}if strings. Contains (FI. Name (), "Lost+found") {Continue}go Fm.walkdir (Fpath)} else {FM. Dirchan <-Fpath}//log. Println ("Path:", Fpath)//ch <-FPATH}//}FUNC (FM *FM) Lister () {var path stringfor {Select {case path = &LT;-FM. DIRCHAN:ERR: = FM. Watcher.add (PATH) if err! = Nil {log. Fatal ("Add Watcher Error:", err)}}}}FUNC (FM *FM) Start () {GO FM.WALKDIR (FM. Basedir) go Fm.lister () defer FM. Watcher.close () <-fm. WDONE}FUNC (FM *FM) process_event (event fsnotify. Event) {Switch event. Op {case Fsnotify. Create:fm. Watcher.add (event. Name) log. Println ("Create:", event.) Name) Case Fsnotify. Rename, Fsnotify. Remove:log. Println ("Remove:", event.) NAME) FM. Watcher.remove (event. Name) Case Fsnotify. Write:log. Println ("Write:", event.) Name)}}func Main () {runtime. Gomaxprocs (runtime. NUMCPU ()/2)/* echo 50000000 >/proc/sys/fs/inotify/max_user_watches echo 327679 >/proc/sys/fs/inotify/max_queue D_events*/filem: = new (Fm) Filem. Init (OS. ARGS[1]) Filem. Start ()}

The interesting part of the above program is that the recursive directory, with multi-threaded communication, but the directory and file a lot of time, the resulting thread is also very large.

Here is a simple example, this example can work, the speed is good, 140G small file directory, about 1.7G virtual memory, about 500m, due to and sersync performance difference too far, so temporarily abandoned this monitoring use.

Main.gopackage mainimport ("FMT" "Log" "OS" "Path/filepath" "github.com/fsnotify/fsnotify")//type Mywatcher * Fsnotify. Watcherfunc Doev (Watcher *fsnotify. Watcher, event fsnotify. Event) {Switch event. Op {case Fsnotify. Create:watcher. Add (event. Name) log. Println ("Create:", event.) Name) Case Fsnotify. Rename, Fsnotify. Remove:log. Println ("Remove:", event.) Name) Watcher. Remove (event. Name) Case Fsnotify. Write:log. Println ("Write:", event.) Name)}}func Main () {watchdir: = os. Args[1]var err Errorwatcher, err: = Fsnotify. Newwatcher () if err! = Nil {log. Fatal (Err)}defer watcher. Close () Done: = Make (chan bool) go func () {for {select {case event: = <-watcher. Events://log. Println ("Event:", event) Doev (Watcher, event) Case ERR: = <-watcher. Errors:log. Println ("Error:", Err)}}} () Err = Watcher. ADD (WATCHDIR) if err! = Nil {log. Fatal (ERR)}err = filepath. Walk (Watchdir, func (path string, info OS. FileInfo, err Error) error {err = Watcher. ADD (PATH) if err! = Nil {log. Fatal (ERR)}return nil}) if err! = Nil {fmt. Printf ("WalkError [%v]\n ", err)}<-done}