| | |
| | | using System; |
| | | using System.Collections.Generic; |
| | | using System.Globalization; |
| | | using System.IO; |
| | | using System.IO.Compression; |
| | | using System.Linq; |
| | | using System.Net; |
| | | using System.Net.Http; |
| | | using System.Net.Http.Headers; |
| | | using System.Text; |
| | |
| | | |
| | | |
| | | //采购公告 |
| | | public static void caigougonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page, string cgfs, string cgfsName) |
| | | public static async void caigougonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page, string cgfs, string cgfsName) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | try |
| | | { |
| | | using (HttpClient client = new HttpClient()) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | } |
| | | } |
| | | else |
| | | { |
| | | // 处理请求失败的情况 |
| | | } |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | |
| | | |
| | | |
| | | //意向公开 |
| | | public static void yixianggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime,string endTime, int page) |
| | | public static async void yixianggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime,string endTime, int page) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | try |
| | | { |
| | | using (HttpClient client = new HttpClient()) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | } |
| | | } |
| | | else |
| | | { |
| | | // 处理请求失败的情况 |
| | | } |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | |
| | | |
| | | |
| | | //结果公告 |
| | | public static void zhongbiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | public static async void zhongbiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | try |
| | | { |
| | | using (HttpClient client = new HttpClient()) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | } |
| | | } |
| | | else |
| | | { |
| | | // 处理请求失败的情况 |
| | | } |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | |
| | | } |
| | | |
| | | //更正公告 |
| | | public static void gengzhenggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | public static async void gengzhenggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | try |
| | | { |
| | | using (HttpClient client = new HttpClient()) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | } |
| | | } |
| | | else |
| | | { |
| | | // 处理请求失败的情况 |
| | | } |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | |
| | | |
| | | |
| | | //废标公告 |
| | | public static void feibiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | public static async void feibiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | try |
| | | { |
| | | using (HttpClient client = new HttpClient()) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | } |
| | | } |
| | | else |
| | | { |
| | | // 处理请求失败的情况 |
| | | } |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |