C # Large file reads and queries--memory mapping

I recently need to quickly query log files , file size of more than 4G.

The requirements are as follows:

1. Read the specified line in a file about 4G or so, the program runs in memory not exceeding 500M.

2. Want to query 1G to content, can control around 20s.

At first I thought this should be easy . after a day of research , discover this requires the use of memory-mapping techniques.

After reviewing the relevant information

https://msdn.microsoft.com/zh-cn/library/dd997372 (v=vs.110). aspx?cs-save-lang=1&cs-lang=csharp# Code-snippet-1

The discovery still has some complexity. In particular, character handling is required.

I wrote a demo myself, hoping to achieve

Unfortunately , the test results , query about 1G of content , spend time in 100s around .

The procedure is as follows:

usingSystem;usingSystem.IO;usingSystem.IO.MemoryMappedFiles;usingSystem.Text;namespaceconsoledemo{classProgram {Private Const stringTxt_file_path =@"e:\ Open source learning \ oversize text file read \file\a.txt"; Private Const stringSplit_varchar ="embarrassing"; Private Const CharSplit_char ='embarrassing'; Private Static LongFile_size =0; Static voidMain (string[] args) {            //long ttargetrownum = 39999999;            LongTtargetrownum =10000000; DateTime BeginTime=DateTime.Now; stringline =Creatememorymapfile (Ttargetrownum); DoubleTotalSeconds =DateTime.Now.Subtract (beginTime).            TotalSeconds;            Console.WriteLine (line); Console.WriteLine (string. Format ("find the {0} line, total time: {1}s", Ttargetrownum, totalseconds));        Console.ReadLine (); }        /// <summary>        ///Create a memory-mapped file/// </summary>        Private Static stringCreatememorymapfile (Longttargetrownum) {            stringline =string.            Empty; using(FileStream fs =NewFileStream (Txt_file_path, FileMode.Open, FileAccess.ReadWrite)) {                LongTargetrownum = Ttargetrownum +1;//Target Line                LongCurrownum =1;//Current LineFile_size =FS.                Length; using(MemoryMappedFile MMF = Memorymappedfile.createfromfile (FS,"Test"Fs. Length, Memorymappedfileaccess.readwrite,NULL, Handleinheritability.none,false))                {                    Longoffset =0; //int limit =;                    intLimit = $; Try{StringBuilder Sbdefinerowline=NewStringBuilder ();  Do                        {                            LongRemaining = fs. Length-offset; using(Memorymappedviewstream Mmstream = mmf.) Createviewstream (offset, remaining > limit?limit:remaining)) //using (Memorymappedviewstream mmstream = mmf. Createviewstream (offset, remaining)){offset+=limit; using(StreamReader sr =NewStreamReader (Mmstream)) {                                    //string ss = Sr. ReadToEnd (). ToString (). Replace ("\ n", "embarrassed"). Replace (Environment.NewLine, "embarrassing");                                    stringSS = Sr. ReadToEnd (). ToString (). Replace ("\ n", Split_varchar).                                    Replace (Environment.NewLine, Split_varchar); if(Currownum <=targetrownum) {                                        if(Currownum <targetrownum) {                                            strings =sbdefinerowline.tostring (); intpos =S.lastindexof (Split_char); if(Pos >0) Sbdefinerowline.remove (0, POS); }                                        Else{ Line=sbdefinerowline.tostring (); returnLine ; }                                        if(ss. Contains (Split_varchar)) {Currownum +=GETNEWLINENUMSOFSTR (ss);                                        Sbdefinerowline.append (ss); }                                        Else{sbdefinerowline.append (ss); }                                    }                                    //Sbdefinerowline.append (ss); //Line = sbdefinerowline.tostring (); //if (ss. Contains (Environment.NewLine))//{                                    //++currownum; //    //currownum++; //    //Currownum + = GETNEWLINENUMSOFSTR (ss); //    //Sbdefinerowline.append (ss); //}                                    //if (currownum = = targetrownum)//{                                    //string s = ""; //}Sr.                                Dispose ();                            } mmstream.dispose (); }                        }  while(Offset <FS.                    Length); }                    Catch(Exception e) {Console.WriteLine (e.message); }                    returnLine ; }            }        }        Private Static LongGETNEWLINENUMSOFSTR (strings) {string[] _lst =S.split (Split_char); return_lst. Length-1; }    }}

View Code

Test:

Welcome you to provide a better solution to the idea.

Resources:

https://msdn.microsoft.com/zh-cn/library/dd997372 (v=vs.110). aspx?cs-save-lang=1&cs-lang=csharp# Code-snippet-1

http://blog.csdn.net/onejune2013/article/details/7577152

C # Large file reads and queries--memory mapping