c# HTTPHelper
发布日期:2021-05-09 00:53:50 浏览次数:17 分类:博客文章

本文共 8664 字,大约阅读时间需要 28 分钟。

c# 爬虫常用的3个方法,备份一下

1 using System;  2 using System.Collections.Generic;  3 using System.IO;  4 using System.Linq;  5 using System.Net;  6 using System.Text;  7   8 namespace MiSuMi  9 { 10     public class HttpHelper 11     { 12         public CookieContainer cookie; 13         public HttpHelper() 14         { 15             cookie = new CookieContainer(); 16         } 17  18         public string Get_Request( 19             string strUrl, 20             CookieContainer _cookie = null, 21             string strHost = "", 22             string strRefer = "", 23             string strOrigin = "", 24             Dictionary
lstHeads = null, 25 string strEncoding = "utf-8", 26 string strContentType = "", 27 string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 28 string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36", 29 bool blnAllowAutoRedirect = true, 30 int intTimeout = 1000 * 30) 31 { 32 HttpWebRequest request; 33 HttpWebResponse response; 34 request = (HttpWebRequest)WebRequest.Create(strUrl); 35 request.Accept = strAccept; 36 request.Timeout = intTimeout; 37 request.Method = "GET"; 38 request.Credentials = CredentialCache.DefaultCredentials; 39 request.UserAgent = strUserAgent; 40 request.AllowAutoRedirect = blnAllowAutoRedirect; 41 if (!string.IsNullOrEmpty(strContentType)) 42 { 43 request.ContentType = strContentType; 44 } 45 if (_cookie != null) 46 { 47 request.CookieContainer = _cookie; 48 } 49 if (!string.IsNullOrEmpty(strHost)) 50 { 51 request.Host = strHost; 52 } 53 if (!string.IsNullOrEmpty(strRefer)) 54 { 55 request.Referer = strRefer; 56 } 57 if (!string.IsNullOrEmpty(strOrigin)) 58 { 59 request.Headers.Add("Origin", strOrigin); 60 } 61 if (lstHeads != null && lstHeads.Count > 0) 62 { 63 foreach (var item in lstHeads) 64 { 65 request.Headers.Add(item.Key, item.Value); 66 } 67 } 68 response = (HttpWebResponse)request.GetResponse(); 69 var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding)); 70 string strResult = sr.ReadToEnd(); 71 sr.Close(); 72 request.Abort(); 73 response.Close(); 74 return strResult; 75 76 } 77 78 public string POST_Request( 79 string strUrl, 80 string postDataStr, 81 CookieContainer _cookie = null, 82 string strHost = "", 83 string strRefer = "", 84 string strOrigin = "", 85 Dictionary
lstHeads = null, 86 string strEncoding = "utf-8", 87 string strContentType = "", 88 string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 89 string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36", 90 bool blnAllowAutoRedirect = true, 91 int intTimeout = 1000 * 30) 92 { 93 HttpWebRequest request; 94 HttpWebResponse response; 95 request = (HttpWebRequest)WebRequest.Create(strUrl); 96 request.Accept = strAccept; 97 request.Timeout = intTimeout; 98 request.Method = "POST"; 99 request.Host = strHost;100 request.UserAgent = strUserAgent;101 if (_cookie != null)102 {103 request.CookieContainer = _cookie;104 }105 request.AllowAutoRedirect = blnAllowAutoRedirect;106 if (!string.IsNullOrEmpty(strContentType))107 {108 request.ContentType = strContentType;109 }110 if (!string.IsNullOrEmpty(strOrigin))111 {112 request.Headers.Add("Origin", strOrigin);113 }114 if (!string.IsNullOrEmpty(strRefer))115 {116 request.Referer = strRefer;117 }118 if (!string.IsNullOrEmpty(strHost))119 {120 request.Host = strHost;121 }122 if (lstHeads != null && lstHeads.Count > 0)123 {124 foreach (var item in lstHeads)125 {126 request.Headers.Add(item.Key, item.Value);127 }128 }129 if (!string.IsNullOrEmpty(postDataStr))130 {131 request.ContentLength = postDataStr.Length;132 Stream myRequestStream = request.GetRequestStream();133 StreamWriter myStreamWriter = new StreamWriter(myRequestStream);134 myStreamWriter.Write(postDataStr);135 myStreamWriter.Close();136 }137 138 response = (HttpWebResponse)request.GetResponse();139 var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));140 string strResult = sr.ReadToEnd();141 sr.Close();142 request.Abort();143 response.Close();144 return strResult;145 }146 147 private string DownloadFile(148 string strURLAddress,149 string strPath,150 CookieContainer _cookie = null,151 string strHost = "",152 string strRefer = "",153 string strOrigin = "",154 Dictionary
lstHeads = null,155 string strAccept = "",156 string strUserAgent = "")157 {158 try159 {160 // 设置参数161 HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest;162 if (!string.IsNullOrEmpty(strAccept))163 {164 request.Accept = strAccept;165 }166 if (!string.IsNullOrEmpty(strUserAgent))167 {168 request.UserAgent = strUserAgent;169 }170 if (_cookie != null)171 {172 request.CookieContainer = _cookie;173 }174 if (!string.IsNullOrEmpty(strOrigin))175 {176 request.Headers.Add("Origin", strOrigin);177 }178 if (!string.IsNullOrEmpty(strRefer))179 {180 request.Referer = strRefer;181 }182 if (!string.IsNullOrEmpty(strHost))183 {184 request.Host = strHost;185 }186 if (lstHeads != null && lstHeads.Count > 0)187 {188 foreach (var item in lstHeads)189 {190 request.Headers.Add(item.Key, item.Value);191 }192 }193 //发送请求并获取相应回应数据194 HttpWebResponse response = request.GetResponse() as HttpWebResponse;195 string strReceivePath = string.Empty;196 197 //直到request.GetResponse()程序才开始向目标网页发送Post请求198 Stream responseStream = response.GetResponseStream();199 //创建本地文件写入流200 Stream stream = new FileStream(strPath, FileMode.Create);201 byte[] bArr = new byte[1024];202 int size = responseStream.Read(bArr, 0, (int)bArr.Length);203 while (size > 0)204 {205 stream.Write(bArr, 0, size);206 stream.Flush();207 size = responseStream.Read(bArr, 0, (int)bArr.Length);208 }209 stream.Close();210 responseStream.Close();211 return strPath;212 }213 catch (Exception ex)214 {215 return "";216 }217 }218 219 }220 }
View Code

常用的方法,参数就这些了,后面如果有用到其他的再补充了

上一篇:c# json key转大小写
下一篇:Sql一行拆分转多行

发表评论

最新留言

第一次来,支持一个
[***.219.124.196]2025年04月14日 02时11分46秒