Mareo Code

WebClient class is very handy and useful but has one big gap. There is no build in cookies handling between requests. Luckily somebody wrote briliant solution for this :) http://couldbedone.blogspot.com/2007/08/webclient-handling-cookies.html

With above snippet the WebClient class will be enough for most common web tasks.

Below you find my rich (maybe to rich) commented example of use WebClient class (enhanced) to send data(POST, GET) and retrieve response (webpage source). There are also HttpWebRequest/HttpWebResponse classes example so you can compare both methods.

In my examples I ‘m going to connect to http://www.phpbb.com, login into this community site and after success pull some data from Phpbb User Configuration Panel.

Full code available to download: WebClientEnhancedFetchWebpage Example (83)

So in below example we will login into phpbb.com forum.

using System;
using System.Collections.Generic;
using System.Text;
using System.Net;

namespace WebClientEnhancedFetchWebpage
{
    class Program
    {
        static void Main(string[] args)
        {
            string webpageSource = String.Empty;
            byte[] responseBytes;
            string url = String.Empty;
            string pageCharset = String.Empty;
            System.Collections.Specialized.NameValueCollection dataToSend = new System.Collections.Specialized.NameValueCollection();

            #region Send POST/GET HTTP data using WebClient with cookies
            // Easy way and enough in most cases
            using (WebClientEnhanced webClient = new WebClientEnhanced())
            {
                url = "http://www.phpbb.com/community/ucp.php?mode=login";

                // Below custom charset
                //pageCharset =  "ISO-8859-2";
                //webClient.Encoding = Encoding.GetEncoding(pageCharset);

                // Phpbb use UTF-8 so we can use build in property
                // instead some uncommon charset.
                webClient.Encoding = Encoding.UTF8;

                // Prepare data (in this case, login credentials) to send.

                // Variables names are CaseSensitive so must be exactly the same
                // as in webpage source
                dataToSend.Add("username", "ENTER_YOUR_OWN_DATA");
                dataToSend.Add("password", "ENTER_YOUR_OWN_DATA");

                // Often submit button is also nedeed for proper form post
                // so just add it. Also you can add all other fields from
                // webpage form but usually it's not needed.
                dataToSend.Add("login", "Login");

We get fields name from phpbb.com login webpage source.

                // Let's send data by POST method and get the result response
                responseBytes = webClient.UploadValues(url, WebRequestMethods.Http.Post, dataToSend);

                // Response return as byte[] array so let's convert it into ordinary string
                // Here you can check the webpage source by Visual Studio HTMl Visualizer.

                webpageSource = webClient.Encoding.GetString(responseBytes);

                // Ok, we are logged in succesfully!

                // Now because we use WebClient enhanced WITH cookies container
                // so we are able to pull other secured (by session id saved in cookie)
                // webpages.
                // Let's get some profile webpage, for example, User Control Panel
                url = "http://www.phpbb.com/community/ucp.php";

                // Our WebClientEnhanced instance has authorization cookies saved so
                // we can make just simple call.
                // Here you can check the webpage source by Visual Studio HTMl Visualizer.
                webpageSource = webClient.DownloadString(url);
            }
            #endregion

            #region Send POST/GET HTTP data using HttpWebRequest/HttpWebResponse classes
            // More complicated method but sometimes we will need more control
            // then this will be suitable.
            url = "http://www.phpbb.com/community/ucp.php?mode=login";

            // Data to send (login credentials)
            string webUsername = "ENTER_YOUR_OWN_DATA";
            string webPassword = "ENTER_YOUR_OWN_DATA";
            string webSubmit = "Login";

            // Data will be send in format: fieldname=VALUE&fieldname=VALUE
            string sendData = String.Format("username={0}&password={1}&login={2}",
                webUsername, webPassword, webSubmit);

            // This container will keep cookies send by webpage/server after
            // request. So in case of successfull login the session id
            // will be stored here.
            CookieContainer ourCookiesContainer = new CookieContainer();

            // Create a request using a URL that can receive a post.
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            // Assign to the request.CookieContainer property our cookies container
            // to get all cookies from the request (ourCookiesContainer will keep all
            // cookies returned by server)
            request.CookieContainer = ourCookiesContainer;

            // Set the Method property of the request to POST.
            request.Method = WebRequestMethods.Http.Post;

            // Set the ContentType property of the WebRequest.
            request.ContentType = "application/x-www-form-urlencoded";

            // Create POST data and convert it to a byte array.
            byte[] byteArray = Encoding.UTF8.GetBytes(sendData);

            // Set the ContentLength property of the WebRequest.
            request.ContentLength = byteArray.Length;

            // Create the request stream.
            using (System.IO.Stream dataStream = request.GetRequestStream())
            {
                // Write the data to the request stream.
                // In other words, send a data to URL.
                dataStream.Write(byteArray, 0, byteArray.Length);
            }

            // Get the response.
            using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
            {
                // Display the status returned with the response.
                Console.WriteLine(url + "\nServer response = " + response.StatusDescription);

                    // Open the response stream using a StreamReader for easy access.
                    using (System.IO.StreamReader reader = new System.IO.StreamReader(response.GetResponseStream(), System.Text.Encoding.UTF8))
                    {
                        // Read the content into string.
                        // Here you can check the webpage source by Visual Studio HTMl Visualizer.
                        string responseFromServer = reader.ReadToEnd();

                        // Just a exmaple how to read the cookies collection from CookieContainer
                        CookieCollection storedCookies = request.CookieContainer.GetCookies(request.RequestUri);

                        // Take a look on cookies
                        foreach (Cookie item in storedCookies)
                        {
                            Console.WriteLine(String.Format("Cookie name: {0} Value: {1}", item.Name, item.Value));
                        }

                    }
            }

            // Uff, a lot of code in comparision to WebClient
            // but this is not the end yet!

            // We sent a request and get the response. We are logged in and cookies
            // are stored in ourCookiesContainer object.
            // So we are ready to send second url request for User Config Panel data.

            url = "http://www.phpbb.com/community/ucp.php";

            request = (HttpWebRequest)WebRequest.Create(url);
            // Keep the same cookies bag :)
            request.CookieContainer = ourCookiesContainer;
            request.Method = WebRequestMethods.Http.Get;

            using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
            {
                Console.WriteLine(url + "\nServer response = " + response.StatusDescription);

                using (System.IO.StreamReader reader = new System.IO.StreamReader(response.GetResponseStream(), System.Text.Encoding.UTF8))
                {
                    string responseFromServer = reader.ReadToEnd();
                }

            }
            #endregion

            Console.ReadKey();
        }
    }
}