How to make crawler in C#

   class Program
    {
        private static string GetWebText(string url)
        {

            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);

            request.UserAgent = "A .NET Web Crawler";
            WebResponse response = request.GetResponse();
            Stream stream = response.GetResponseStream();
            StreamReader reader = new StreamReader(stream);
            string htmlText = reader.ReadToEnd();
            return htmlText;

        }

        static void Main(string[] args)
        {
            string str=GetWebText("http://www.rategain.com");
            Console.WriteLine(str);
            FileStream fs = new FileStream("d:/abc.html", FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None);
            StreamWriter wr = new StreamWriter(fs);
            wr.Write(str);
        }
     
    }