【轉】C#大檔案讀取和查詢--記憶體映射

標籤：asp   empty   create   .com   path   html   lang   index   mat   

筆者最近需要快速查詢記錄檔,檔案大小在4G以上。

需求如下:

1.讀取4G左右大小的檔案中的指定行，程式運行佔用記憶體不超過500M。

2.希望查詢1G以內容,能控制在20s左右.

剛開始覺得這個應該不難.研究一天之後,發現這個需要使用記憶體映射技術。

查閱了相關資料之後

https://msdn.microsoft.com/zh-cn/library/dd997372(v=vs.110).aspx?cs-save-lang=1&cs-lang=csharp#code-snippet-1

發現還是有一定的複雜性.特別是需要對字元處理。

筆者自己寫了一個Demo,希望實現

很遺憾,測試結果,查詢1G左右的內容,花費時間在100s左右.

程式如下:

using System;using System.IO;using System.IO.MemoryMappedFiles;using System.Text;namespace ConsoleDemo{    class Program    {        private const string TXT_FILE_PATH = @"E:\開源學習\超大文字檔讀取\File\a.txt";        private const string SPLIT_VARCHAR = "囧";        private const char SPLIT_CHAR = ‘囧‘;        private static long FILE_SIZE = 0;        static void Main(string[] args)        {            //long ttargetRowNum = 39999999;            long ttargetRowNum = 10000000;            DateTime beginTime = DateTime.Now;            string line = CreateMemoryMapFile(ttargetRowNum);            double totalSeconds = DateTime.Now.Subtract(beginTime).TotalSeconds;            Console.WriteLine(line);            Console.WriteLine(string.Format("尋找第{0}行,共耗時:{1}s", ttargetRowNum, totalSeconds));            Console.ReadLine();        }        /// <summary>        /// 建立記憶體對應檔        /// </summary>        private static string CreateMemoryMapFile(long ttargetRowNum)        {            string line = string.Empty;            using (FileStream fs = new FileStream(TXT_FILE_PATH, FileMode.Open, FileAccess.ReadWrite))            {                long targetRowNum = ttargetRowNum + 1;//目標行                long curRowNum = 1;//當前行                FILE_SIZE = fs.Length;                using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(fs, "test", fs.Length, MemoryMappedFileAccess.ReadWrite, null, HandleInheritability.None, false))                {                    long offset = 0;                    //int limit = 250;                    int limit = 200;                    try                    {                        StringBuilder sbDefineRowLine = new StringBuilder();                        do                        {                            long remaining = fs.Length - offset;                            using (MemoryMappedViewStream mmStream = mmf.CreateViewStream(offset, remaining > limit ? limit : remaining))                            //using (MemoryMappedViewStream mmStream = mmf.CreateViewStream(offset, remaining))                            {                                offset += limit;                                using (StreamReader sr = new StreamReader(mmStream))                                {                                    //string ss = sr.ReadToEnd().ToString().Replace("\n", "囧").Replace(Environment.NewLine, "囧");                                    string ss = sr.ReadToEnd().ToString().Replace("\n", SPLIT_VARCHAR).Replace(Environment.NewLine, SPLIT_VARCHAR);                                    if (curRowNum <= targetRowNum)                                    {                                        if (curRowNum < targetRowNum)                                        {                                            string s = sbDefineRowLine.ToString();                                            int pos = s.LastIndexOf(SPLIT_CHAR);                                            if (pos > 0)                                                sbDefineRowLine.Remove(0, pos);                                        }                                        else                                        {                                            line = sbDefineRowLine.ToString();                                            return line;                                        }                                        if (ss.Contains(SPLIT_VARCHAR))                                        {                                            curRowNum += GetNewLineNumsOfStr(ss);                                            sbDefineRowLine.Append(ss);                                        }                                        else                                        {                                            sbDefineRowLine.Append(ss);                                        }                                    }                                    //sbDefineRowLine.Append(ss);                                    //line = sbDefineRowLine.ToString();                                    //if (ss.Contains(Environment.NewLine))                                    //{                                    //    ++curRowNum;                                    //    //curRowNum++;                                    //    //curRowNum += GetNewLineNumsOfStr(ss);                                    //    //sbDefineRowLine.Append(ss);                                    //}                                    //if (curRowNum == targetRowNum)                                    //{                                    //    string s = "";                                    //}                                    sr.Dispose();                                }                                mmStream.Dispose();                            }                        } while (offset < fs.Length);                    }                    catch (Exception e)                    {                        Console.WriteLine(e.Message);                    }                    return line;                }            }        }        private static long GetNewLineNumsOfStr(string s)        {            string[] _lst = s.Split(SPLIT_CHAR);            return _lst.Length - 1;        }    }}

　　

 

歡迎大家提供更好的解決思路.

參考資料:

https://msdn.microsoft.com/zh-cn/library/dd997372(v=vs.110).aspx?cs-save-lang=1&cs-lang=csharp#code-snippet-1

http://blog.csdn.net/onejune2013/article/details/7577152

轉自：http://www.cnblogs.com/lucky_hu/p/5345423.html

【轉】C#大檔案讀取和查詢--記憶體映射