username@email.com
2023-10-07 52267147e624f3a0daef4870ba72f023ef9162a9
修改中国政府采购网爬虫规则
1个文件已修改
153 ■■■■■ 已修改文件
zhengcaioa/Crawler/sichuan/CcgpSichuanoperation.cs 153 ●●●●● 补丁 | 查看 | 原始文档 | blame | 历史
zhengcaioa/Crawler/sichuan/CcgpSichuanoperation.cs
@@ -3,7 +3,10 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
@@ -171,7 +174,7 @@
        //采购公告
        public static void caigougonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page, string cgfs, string cgfsName)
        public static async void caigougonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page, string cgfs, string cgfsName)
        {
            sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -185,12 +188,36 @@
                string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
                try
                {
                    using (HttpClient client = new HttpClient())
                    HttpClientHandler handler = new HttpClientHandler();
                    handler.CookieContainer = new CookieContainer();
                    using (HttpClient client = new HttpClient(handler))
                    {
                        client.Timeout = TimeSpan.FromSeconds(60);
                        client.DefaultRequestHeaders.Add("Accept", "*/*");
                        client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
                        client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
                        //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
                        HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
                        var res = response.Content.ReadAsStringAsync().Result;
                        string res = "";
                        if (response.IsSuccessStatusCode)
                        {
                            using (var responseStream = await response.Content.ReadAsStreamAsync())
                            {
                                using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
                                {
                                    using (var reader = new StreamReader(decompressedStream))
                                    {
                                        res = await reader.ReadToEndAsync();
                                        // 处理解压缩后的响应内容
                                    }
                                }
                            }
                        }
                        else
                        {
                            // 处理请求失败的情况
                        }
                        //var res = response.Content.ReadAsStringAsync().Result;
                        var document = parser.ParseDocument(res);
                        var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
                        var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -542,7 +569,7 @@
        //意向公开
        public static void yixianggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime,string endTime, int page)
        public static async void yixianggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime,string endTime, int page)
        {
            sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -556,12 +583,36 @@
                string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
                try
                {
                    using (HttpClient client = new HttpClient())
                    HttpClientHandler handler = new HttpClientHandler();
                    handler.CookieContainer = new CookieContainer();
                    using (HttpClient client = new HttpClient(handler))
                    {
                        client.Timeout = TimeSpan.FromSeconds(60);
                        client.DefaultRequestHeaders.Add("Accept", "*/*");
                        client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
                        client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
                        //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
                        HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
                        var res = response.Content.ReadAsStringAsync().Result;
                        //var res = response.Content.ReadAsStringAsync().Result;
                        string res = "";
                        if (response.IsSuccessStatusCode)
                        {
                            using (var responseStream = await response.Content.ReadAsStreamAsync())
                            {
                                using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
                                {
                                    using (var reader = new StreamReader(decompressedStream))
                                    {
                                        res = await reader.ReadToEndAsync();
                                        // 处理解压缩后的响应内容
                                    }
                                }
                            }
                        }
                        else
                        {
                            // 处理请求失败的情况
                        }
                        var document = parser.ParseDocument(res);
                        var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
                        var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -857,7 +908,7 @@
        //结果公告
        public static void zhongbiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
        public static async void zhongbiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
        {
            sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -871,12 +922,36 @@
                string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
                try
                {
                    using (HttpClient client = new HttpClient())
                    HttpClientHandler handler = new HttpClientHandler();
                    handler.CookieContainer = new CookieContainer();
                    using (HttpClient client = new HttpClient(handler))
                    {
                        client.Timeout = TimeSpan.FromSeconds(60);
                        client.DefaultRequestHeaders.Add("Accept", "*/*");
                        client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
                        client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
                        //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
                        HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
                        var res = response.Content.ReadAsStringAsync().Result;
                        //var res = response.Content.ReadAsStringAsync().Result;
                        string res = "";
                        if (response.IsSuccessStatusCode)
                        {
                            using (var responseStream = await response.Content.ReadAsStreamAsync())
                            {
                                using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
                                {
                                    using (var reader = new StreamReader(decompressedStream))
                                    {
                                        res = await reader.ReadToEndAsync();
                                        // 处理解压缩后的响应内容
                                    }
                                }
                            }
                        }
                        else
                        {
                            // 处理请求失败的情况
                        }
                        var document = parser.ParseDocument(res);
                        var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
                        var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -1224,7 +1299,7 @@
        }
        //更正公告
        public static void gengzhenggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
        public static async void gengzhenggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
        {
            sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -1238,12 +1313,36 @@
                string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
                try
                {
                    using (HttpClient client = new HttpClient())
                    HttpClientHandler handler = new HttpClientHandler();
                    handler.CookieContainer = new CookieContainer();
                    using (HttpClient client = new HttpClient(handler))
                    {
                        client.Timeout = TimeSpan.FromSeconds(60);
                        client.DefaultRequestHeaders.Add("Accept", "*/*");
                        client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
                        client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
                        //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
                        HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
                        var res = response.Content.ReadAsStringAsync().Result;
                        //var res = response.Content.ReadAsStringAsync().Result;
                        string res = "";
                        if (response.IsSuccessStatusCode)
                        {
                            using (var responseStream = await response.Content.ReadAsStreamAsync())
                            {
                                using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
                                {
                                    using (var reader = new StreamReader(decompressedStream))
                                    {
                                        res = await reader.ReadToEndAsync();
                                        // 处理解压缩后的响应内容
                                    }
                                }
                            }
                        }
                        else
                        {
                            // 处理请求失败的情况
                        }
                        var document = parser.ParseDocument(res);
                        var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
                        var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -1539,7 +1638,7 @@
        //废标公告
        public static void feibiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
        public static async void feibiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
        {
            sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -1553,12 +1652,36 @@
                string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
                try
                {
                    using (HttpClient client = new HttpClient())
                    HttpClientHandler handler = new HttpClientHandler();
                    handler.CookieContainer = new CookieContainer();
                    using (HttpClient client = new HttpClient(handler))
                    {
                        //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
                        client.Timeout = TimeSpan.FromSeconds(60);
                        client.DefaultRequestHeaders.Add("Accept", "*/*");
                        client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
                        client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
                        HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
                        var res = response.Content.ReadAsStringAsync().Result;
                        //var res = response.Content.ReadAsStringAsync().Result;
                        string res = "";
                        if (response.IsSuccessStatusCode)
                        {
                            using (var responseStream = await response.Content.ReadAsStreamAsync())
                            {
                                using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
                                {
                                    using (var reader = new StreamReader(decompressedStream))
                                    {
                                        res = await reader.ReadToEndAsync();
                                        // 处理解压缩后的响应内容
                                    }
                                }
                            }
                        }
                        else
                        {
                            // 处理请求失败的情况
                        }
                        var document = parser.ParseDocument(res);
                        var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
                        var contentList = sssdfsdfsd.QuerySelector("ul");