
c# HTTPHelper
View Code
发布日期:2021-05-09 00:53:50
浏览次数:17
分类:博客文章
本文共 8664 字,大约阅读时间需要 28 分钟。
c# 爬虫常用的3个方法,备份一下
1 using System; 2 using System.Collections.Generic; 3 using System.IO; 4 using System.Linq; 5 using System.Net; 6 using System.Text; 7 8 namespace MiSuMi 9 { 10 public class HttpHelper 11 { 12 public CookieContainer cookie; 13 public HttpHelper() 14 { 15 cookie = new CookieContainer(); 16 } 17 18 public string Get_Request( 19 string strUrl, 20 CookieContainer _cookie = null, 21 string strHost = "", 22 string strRefer = "", 23 string strOrigin = "", 24 DictionarylstHeads = null, 25 string strEncoding = "utf-8", 26 string strContentType = "", 27 string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 28 string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36", 29 bool blnAllowAutoRedirect = true, 30 int intTimeout = 1000 * 30) 31 { 32 HttpWebRequest request; 33 HttpWebResponse response; 34 request = (HttpWebRequest)WebRequest.Create(strUrl); 35 request.Accept = strAccept; 36 request.Timeout = intTimeout; 37 request.Method = "GET"; 38 request.Credentials = CredentialCache.DefaultCredentials; 39 request.UserAgent = strUserAgent; 40 request.AllowAutoRedirect = blnAllowAutoRedirect; 41 if (!string.IsNullOrEmpty(strContentType)) 42 { 43 request.ContentType = strContentType; 44 } 45 if (_cookie != null) 46 { 47 request.CookieContainer = _cookie; 48 } 49 if (!string.IsNullOrEmpty(strHost)) 50 { 51 request.Host = strHost; 52 } 53 if (!string.IsNullOrEmpty(strRefer)) 54 { 55 request.Referer = strRefer; 56 } 57 if (!string.IsNullOrEmpty(strOrigin)) 58 { 59 request.Headers.Add("Origin", strOrigin); 60 } 61 if (lstHeads != null && lstHeads.Count > 0) 62 { 63 foreach (var item in lstHeads) 64 { 65 request.Headers.Add(item.Key, item.Value); 66 } 67 } 68 response = (HttpWebResponse)request.GetResponse(); 69 var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding)); 70 string strResult = sr.ReadToEnd(); 71 sr.Close(); 72 request.Abort(); 73 response.Close(); 74 return strResult; 75 76 } 77 78 public string POST_Request( 79 string strUrl, 80 string postDataStr, 81 CookieContainer _cookie = null, 82 string strHost = "", 83 string strRefer = "", 84 string strOrigin = "", 85 Dictionary lstHeads = null, 86 string strEncoding = "utf-8", 87 string strContentType = "", 88 string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 89 string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36", 90 bool blnAllowAutoRedirect = true, 91 int intTimeout = 1000 * 30) 92 { 93 HttpWebRequest request; 94 HttpWebResponse response; 95 request = (HttpWebRequest)WebRequest.Create(strUrl); 96 request.Accept = strAccept; 97 request.Timeout = intTimeout; 98 request.Method = "POST"; 99 request.Host = strHost;100 request.UserAgent = strUserAgent;101 if (_cookie != null)102 {103 request.CookieContainer = _cookie;104 }105 request.AllowAutoRedirect = blnAllowAutoRedirect;106 if (!string.IsNullOrEmpty(strContentType))107 {108 request.ContentType = strContentType;109 }110 if (!string.IsNullOrEmpty(strOrigin))111 {112 request.Headers.Add("Origin", strOrigin);113 }114 if (!string.IsNullOrEmpty(strRefer))115 {116 request.Referer = strRefer;117 }118 if (!string.IsNullOrEmpty(strHost))119 {120 request.Host = strHost;121 }122 if (lstHeads != null && lstHeads.Count > 0)123 {124 foreach (var item in lstHeads)125 {126 request.Headers.Add(item.Key, item.Value);127 }128 }129 if (!string.IsNullOrEmpty(postDataStr))130 {131 request.ContentLength = postDataStr.Length;132 Stream myRequestStream = request.GetRequestStream();133 StreamWriter myStreamWriter = new StreamWriter(myRequestStream);134 myStreamWriter.Write(postDataStr);135 myStreamWriter.Close();136 }137 138 response = (HttpWebResponse)request.GetResponse();139 var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));140 string strResult = sr.ReadToEnd();141 sr.Close();142 request.Abort();143 response.Close();144 return strResult;145 }146 147 private string DownloadFile(148 string strURLAddress,149 string strPath,150 CookieContainer _cookie = null,151 string strHost = "",152 string strRefer = "",153 string strOrigin = "",154 Dictionary lstHeads = null,155 string strAccept = "",156 string strUserAgent = "")157 {158 try159 {160 // 设置参数161 HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest;162 if (!string.IsNullOrEmpty(strAccept))163 {164 request.Accept = strAccept;165 }166 if (!string.IsNullOrEmpty(strUserAgent))167 {168 request.UserAgent = strUserAgent;169 }170 if (_cookie != null)171 {172 request.CookieContainer = _cookie;173 }174 if (!string.IsNullOrEmpty(strOrigin))175 {176 request.Headers.Add("Origin", strOrigin);177 }178 if (!string.IsNullOrEmpty(strRefer))179 {180 request.Referer = strRefer;181 }182 if (!string.IsNullOrEmpty(strHost))183 {184 request.Host = strHost;185 }186 if (lstHeads != null && lstHeads.Count > 0)187 {188 foreach (var item in lstHeads)189 {190 request.Headers.Add(item.Key, item.Value);191 }192 }193 //发送请求并获取相应回应数据194 HttpWebResponse response = request.GetResponse() as HttpWebResponse;195 string strReceivePath = string.Empty;196 197 //直到request.GetResponse()程序才开始向目标网页发送Post请求198 Stream responseStream = response.GetResponseStream();199 //创建本地文件写入流200 Stream stream = new FileStream(strPath, FileMode.Create);201 byte[] bArr = new byte[1024];202 int size = responseStream.Read(bArr, 0, (int)bArr.Length);203 while (size > 0)204 {205 stream.Write(bArr, 0, size);206 stream.Flush();207 size = responseStream.Read(bArr, 0, (int)bArr.Length);208 }209 stream.Close();210 responseStream.Close();211 return strPath;212 }213 catch (Exception ex)214 {215 return "";216 }217 }218 219 }220 }
常用的方法,参数就这些了,后面如果有用到其他的再补充了
发表评论
最新留言
第一次来,支持一个
[***.219.124.196]2025年04月14日 02时11分46秒
关于作者

喝酒易醉,品茶养心,人生如梦,品茶悟道,何以解忧?唯有杜康!
-- 愿君每日到此一游!
推荐文章
Mybatis Generator最完整配置详解
2019-03-06
[白话解析] 深入浅出熵的概念 & 决策树之ID3算法
2019-03-06
[梁山好汉说IT] 梁山好汉和抢劫银行
2019-03-06
[源码解析] 消息队列 Kombu 之 基本架构
2019-03-06
[源码分析] 消息队列 Kombu 之 启动过程
2019-03-06
[源码分析] 消息队列 Kombu 之 Consumer
2019-03-06
抉择之苦
2019-03-06
wx.NET CLI wrapper for wxWidgets
2019-03-06
ASP.NET MVC Action Filters
2019-03-06
Powershell中禁止执行脚本解决办法
2019-03-06
HTTP协议状态码详解(HTTP Status Code)
2019-03-06
OO_Unit2 多线程电梯总结
2019-03-06
04_Mysql配置文件(重要参数)
2019-03-06
python 序列化及其相关模块(json,pickle,shelve,xml)详解
2019-03-06
JavaSE总结
2019-03-06
手动造轮子——基于.NetCore的RPC框架DotNetCoreRpc
2019-03-06