C# 模拟浏览器发起http Get请求,Post请求,支持GZIP响应体

场景说明

当需要调用接口、下载其它网页数据,可以使用第三方组件(例如RestSharp),但我只想简单用一下,以下主要是自行封装的示例代码。容易漏的是对C# 非托管代码的资源的释放(例如:StreamReader.Close、HttpWebRequest.Abort、Stream.Dispose),并且支持GZIP响应体解析。网络上有很多种实现,但我依然要分享我用了几年的封装,因为多数代码没有注意非托管代码的资源的释放,这样的代码运行久了容易内存泄漏。另外就是对Url的编码需要自行按参数去处理,很麻烦,干脆也封装一下。

代码如下

using System.Text;
using System.Net;
using System.IO;
using System.IO.Compression;

public class ToolX
{
    /// <summary>
    /// 发起Get请求,并接收响应体为字符串
    /// </summary>
    /// <param name="url">网址</param>
    /// <param name="encode">编码格式</param>
    /// <param name="dicHeader">请求头</param>
    /// <returns></returns>
	public string HttpGet(string url, Encoding encode = null, Dictionary<string, string> dicHeader = null) 
	{
        string retStr = string.Empty;
        encode = encode ?? Encoding.UTF8;
        try
        {
            HttpWebRequest hwr = (HttpWebRequest)WebRequest.Create(UrlEncode(url, encode));
            hwr.Method = "GET";
            hwr.ContentType = "text/html; charset=utf-8";
            hwr.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36";
            hwr.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            hwr.KeepAlive = false;
            hwr.Headers.Add("Accept-Encoding", "gzip, deflate");
            if (dicHeader != null)
            {
                foreach (var kv in dicHeader)
                {
                    switch (kv.Key)
                    {
                        //部分请求头不能像这样hwr.Headers[kv.Key] = kv.Value,需要的自行写
                        case "ContentType": hwr.ContentType = kv.Value; break;
                        case "Accept": hwr.Accept = kv.Value; break;
                        case "UserAgent": hwr.UserAgent = kv.Value; break;
                        case "Referer": hwr.Referer = UrlEncode(kv.Value, encode); break;
                        default: hwr.Headers[kv.Key] = kv.Value; break;
                    }
                }
            }

            //下载页面
            HttpWebResponse httpresponse = (HttpWebResponse)hwr.GetResponse();
            retStr = StreamToString(httpresponse.GetResponseStream(), encode, httpresponse.ContentEncoding.ToLower().Trim() == "gzip");
            httpresponse.Close();
            hwr.Abort();//重要,不然会占用TCP链接资源
        }
        catch (WebException ex)
        {
            //日志自行实现
            //ex.Status;  
            var response = ex.Response as System.Net.HttpWebResponse;
            if (response != null)
            {
                //Console.WriteLine("HTTP Status Code: " + (int)response.StatusCode);
                //string errResopnse = StreamToString(response.GetResponseStream(), encode, response.ContentEncoding.ToLower().Trim() == "gzip");
            }
        }
        catch (Exception ex)
		{
            //日志自行实现
		}
		finally
		{
            GC.Collect();
		}

        return retStr;
    }

    /// <summary>
    /// 发起Post请求,并接收响应体为字符串
    /// </summary>
    /// <param name="url">网址</param>
    /// <param name="postBody">请求体,目前只能传字符串、byte[]、Stream</param>
    /// <param name="encode">编码格式</param>
    /// <param name="dicHeader">请求头</param>
    /// <returns></returns>
    public string HttpPost(string url, object postBody, Encoding encode = null, Dictionary<string, string> dicHeader = null)
    {
        string retStr = string.Empty;
        encode = encode ?? Encoding.UTF8;
        try
        {
            HttpWebRequest hwr = (HttpWebRequest)WebRequest.Create(UrlEncode(url, encode));
            hwr.Method = "POST";
            hwr.ContentType = "application/x-www-form-urlencoded";
            hwr.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36";
            hwr.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            hwr.KeepAlive = false;
            hwr.Headers.Add("Accept-Encoding", "gzip, deflate");
            if (dicHeader != null)
            {
                foreach (var kv in dicHeader)
                {
                    switch (kv.Key)
                    {
                        //部分请求头不能像这样hwr.Headers[kv.Key] = kv.Value,需要的自行写
                        case "ContentType": hwr.ContentType = kv.Value; break;
                        case "Accept": hwr.Accept = kv.Value; break;
                        case "UserAgent": hwr.UserAgent = kv.Value; break;
                        case "Referer": hwr.Referer = UrlEncode(kv.Value, encode); break;
                        default: hwr.Headers[kv.Key] = kv.Value; break;
                    }
                }
            }

            Stream postDataStream = hwr.GetRequestStream();
            //处理请求体为byte[]
            byte[] postBytes = null;
            if (postBody != null)
            {
                var tp = postBody.GetType();
                if (tp == typeof(string))
                {
                    encode.GetBytes(postBody + "");
                }
                else if(tp == typeof(byte[]))
				{
                    postBytes = postBody as byte[];
                }
                else if(postBody is Stream)
				{
                    using(Stream sm = postBody as Stream)
					{
                        postBytes = new byte[sm.Length];
                        sm.Position = 0;
                        sm.Read(postBytes, 0, Convert.ToInt32(sm.Length));
					}
				}
                else
				{
                    throw new Exception("不支持的请求体类型");
				}
            }
                
            //发送请求体
            if (postBytes != null)
            {
                hwr.ContentLength = postBytes.Length;
                postDataStream.Write(postBytes, 0, postBytes.Length);
            }

            //下载页面
            HttpWebResponse httpresponse = (HttpWebResponse)hwr.GetResponse();
            retStr = StreamToString(httpresponse.GetResponseStream(), encode, httpresponse.ContentEncoding.ToLower().Trim() == "gzip");
            httpresponse.Close();
            hwr.Abort();//重要,不然会占用TCP链接资源
        }
        catch (WebException ex)
        {
            //日志自行实现
            //ex.Status;  
            var response = ex.Response as System.Net.HttpWebResponse;
            if (response != null)
            {
                //Console.WriteLine("HTTP Status Code: " + (int)response.StatusCode);
                //string errResopnse = StreamToString(response.GetResponseStream(), encode, response.ContentEncoding.ToLower().Trim() == "gzip");
            }
        }
        catch (Exception ex)
        {
            //日志自行实现
        }
        finally
        {
            GC.Collect();
        }

        return retStr;
    }

    /// <summary>
    /// 读取流为字符串
    /// </summary>
    /// <param name="ream">注意,该流会被释放</param>
    /// <param name="encode"></param>
    /// <param name="isGzip"></param>
    /// <returns></returns>
	public string StreamToString(Stream ream, Encoding encode, bool isGzip = false)
	{
        string retStr = null;
        if (ream != null)
        {
            List<byte> body = new List<byte>();
            int b = -1;
            byte[] bts = new byte[102400];
            while ((b = ream.Read(bts, 0, bts.Length)) > 0) body.AddRange(bts.Take(b));
            ream.Close(); ream.Dispose();
            ream = new MemoryStream(body.ToArray());
            if (isGzip)
            {
                ream = new GZipStream(ream, CompressionMode.Decompress);
            }
            StreamReader sr = new StreamReader(ream, encode);
            retStr = sr.ReadToEnd();
            sr.Close(); sr.Dispose(); ream.Close(); ream.Close(); 
        }
        return retStr;
    }


	/// <summary>
	/// 仅对字符大于255的进行编码
	/// </summary>
	/// <param name="url"></param>
	/// <param name="encode"></param>
	/// <returns></returns>
	public string UrlEncode(string url, Encoding encode = null)
    {
        if (string.IsNullOrWhiteSpace(url)) return url;
        encode = encode ?? Encoding.UTF8;
        StringBuilder sb = new StringBuilder();
        foreach (var c in url)
        {
            if (c <= 255) sb.Append(c);
            else sb.Append(System.Web.HttpUtility.UrlEncode("" + c, encode));
        }
        return sb.ToString();
    }
}

调用示例

string ss = new ToolX().HttpGet("https://www.80juchang.com/");
string ss2 = new ToolX().HttpPost("https://www.80juchang.com/", "a=中文");

原文链接:http://blog.albsz.cn/2020-07-17-csharpHttp.html

更多推荐