From 52267147e624f3a0daef4870ba72f023ef9162a9 Mon Sep 17 00:00:00 2001
From: username@email.com <yzy2002yzy@163.com>
Date: 星期六, 07 十月 2023 12:46:29 +0800
Subject: [PATCH] 修改中国政府采购网爬虫规则
---
zhengcaioa/Crawler/sichuan/CcgpSichuanoperation.cs | 510 ++++++++++++++++++++++++++++++++++++--------------------
1 files changed, 329 insertions(+), 181 deletions(-)
diff --git a/zhengcaioa/Crawler/sichuan/CcgpSichuanoperation.cs b/zhengcaioa/Crawler/sichuan/CcgpSichuanoperation.cs
index 53b2a2c..f836bd6 100644
--- a/zhengcaioa/Crawler/sichuan/CcgpSichuanoperation.cs
+++ b/zhengcaioa/Crawler/sichuan/CcgpSichuanoperation.cs
@@ -3,7 +3,10 @@
using System;
using System.Collections.Generic;
using System.Globalization;
+using System.IO;
+using System.IO.Compression;
using System.Linq;
+using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
@@ -19,142 +22,139 @@
{
try
{
+
+
//鑾峰彇鏇存柊鏃堕棿
string operationStartTime = "";
string operationEndTime = "";
- Updatetime updatetime = _ccontext.Updatetimes.Where(x => x.Sheng == "涓浗鏀块噰缃戝洓宸濈渷").FirstOrDefault();
- if (updatetime != null)
+
+ logg.WriteLog("寮�濮嬪垽鏂椂闂�", "涓浗鏀块噰缃戝洓宸濈渷");
+
+ if (DateTime.Now.Hour == 0 )
{
- operationStartTime = updatetime.Updatetime1.ToString("yyyy-MM-dd");
- operationEndTime = DateTime.Now.ToString("yyyy-MM-dd");
- //updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
- //_ccontext.SaveChanges();
+ DateTime datenow = DateTime.Now.Date;
+ Updatetime updatetime = _ccontext.Updatetimes.Where(x => x.Sheng == "涓浗鏀块噰缃戝洓宸濈渷").FirstOrDefault();
+ logg.WriteLog("datenow="+datenow.ToString("yyyy-MM-dd HH:mm:ss"), "涓浗鏀块噰缃戝洓宸濈渷");
+ operationStartTime = datenow.AddDays(-1).ToString("yyyy-MM-dd");
+ operationEndTime = datenow.AddDays(-1).ToString("yyyy-MM-dd");
+ logg.WriteLog("operationStartTime=" + operationStartTime, "涓浗鏀块噰缃戝洓宸濈渷");
+ logg.WriteLog("operationEndTime=" + operationEndTime, "涓浗鏀块噰缃戝洓宸濈渷");
+ var currPage = 1;
+ string sichuanpageurll = "";
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 鎰忓悜鍏紑
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=9&dbselect=bidx&kw=%E6%84%8F%E5%90%91&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鎰忓悜鍏紑 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ yixianggonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鎰忓悜鍏紑 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+
+ #region 鍏紑鎷涙爣
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=1&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 鍏紑鎷涙爣 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "1", "鍏紑鎷涙爣");
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 鍏紑鎷涙爣 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+
+ #region 璇环
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=2&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 璇环 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "4", "璇环");
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 璇环 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 绔炰簤鎬ц皥鍒�
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=3&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 绔炰簤鎬ц皥鍒� 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "5", "绔炰簤鎬ц皥鍒�");
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 绔炰簤鎬ц皥鍒� 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 鍗曚竴鏉ユ簮
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=4&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 鍗曚竴鏉ユ簮 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "7", "鍗曚竴鏉ユ簮");
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 鍗曚竴鏉ユ簮 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 閭�璇锋嫑鏍�
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=6&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 閭�璇锋嫑鏍� 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "2", "閭�璇锋嫑鏍�");
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 閭�璇锋嫑鏍� 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 绔炰簤鎬х鍟�
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=10&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 绔炰簤鎬х鍟� 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "3", "绔炰簤鎬х鍟�");
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 绔炰簤鎬х鍟� 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 涓爣鍏憡
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=7&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷涓爣鍏憡 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ zhongbiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷涓爣鍏憡 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 鎴愪氦鍏憡
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=11&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鎴愪氦鍏憡 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ zhongbiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鎴愪氦鍏憡 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 鏇存鍏憡
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=8&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鏇存鍏憡 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ gengzhenggonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鏇存鍏憡 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+ //Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
+ #region 搴熸爣鍏憡
+ currPage = 1;
+ sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=12&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷搴熸爣鍏憡 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
+ feibiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
+ logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷搴熸爣鍏憡 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
+ #endregion
+
+
+
+ //淇濆瓨鏈�鍚庢洿鏂版椂闂�
+ if (updatetime != null)
+ {
+ updatetime.Updatetime1 = datenow;
+ _ccontext.SaveChanges();
+ }
+ else
+ {
+ updatetime = new Updatetime();
+ updatetime.Id = Guid.NewGuid().ToString();
+ updatetime.Sheng = "涓浗鏀块噰缃戝洓宸濈渷";
+ updatetime.Updatetime1 = datenow;
+ _ccontext.Updatetimes.Add(updatetime);
+ _ccontext.SaveChanges();
+ }
+
+ Thread.CurrentThread.Join(1000 * 60 * 60);//闃绘璁惧畾鏃堕棿
}
- else
- {
- operationStartTime = "2021-12-01"; //DateTime.Now.AddYears(-1).ToString("yyyy-MM-dd");
- operationEndTime = "2022-12-01";//DateTime.Now.ToString("yyyy-MM-dd");
- }
- var currPage = 1;
- string sichuanpageurll = "";
-
- #region 鎰忓悜鍏紑
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=9&dbselect=bidx&kw=%E6%84%8F%E5%90%91&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鎰忓悜鍏紑 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- yixianggonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鎰忓悜鍏紑 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
-
- #region 鍏紑鎷涙爣
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=1&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 鍏紑鎷涙爣 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "1", "鍏紑鎷涙爣");
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 鍏紑鎷涙爣 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
- #region 璇环
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=2&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 璇环 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "4", "璇环");
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 璇环 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
- #region 绔炰簤鎬ц皥鍒�
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=3&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 绔炰簤鎬ц皥鍒� 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "5", "绔炰簤鎬ц皥鍒�");
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 绔炰簤鎬ц皥鍒� 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
- #region 鍗曚竴鏉ユ簮
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=4&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 鍗曚竴鏉ユ簮 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "7", "鍗曚竴鏉ユ簮");
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 鍗曚竴鏉ユ簮 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
- #region 閭�璇锋嫑鏍�
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=6&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 閭�璇锋嫑鏍� 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "2", "閭�璇锋嫑鏍�");
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 閭�璇锋嫑鏍� 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
- #region 绔炰簤鎬х鍟�
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=10&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 绔炰簤鎬х鍟� 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "3", "绔炰簤鎬х鍟�");
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷閲囪喘鍏憡 绔炰簤鎬х鍟� 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
-
- #region 涓爣鍏憡
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=7&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷涓爣鍏憡 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- zhongbiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷涓爣鍏憡 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
- #region 鎴愪氦鍏憡
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=11&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鎴愪氦鍏憡 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- zhongbiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鎴愪氦鍏憡 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
- #region 鏇存鍏憡
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=8&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鏇存鍏憡 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- gengzhenggonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage);
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷鏇存鍏憡 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
- Thread.CurrentThread.Join(1000 * 60 * 30);//闃绘璁惧畾鏃堕棿
-
- #region 搴熸爣鍏憡
- currPage = 1;
- sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=12&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName=";
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷搴熸爣鍏憡 寮�濮嬭幏鍙�", "涓浗鏀块噰缃戝洓宸濈渷");
- feibiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime , currPage);
- logg.WriteLog("涓浗鏀块噰缃戝洓宸濈渷搴熸爣鍏憡 寮�濮嬭幏鍙栫粨鏉�", "涓浗鏀块噰缃戝洓宸濈渷");
- #endregion
-
-
-
- //淇濆瓨鏈�鍚庢洿鏂版椂闂�
- if (updatetime != null)
- {
- updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
- _ccontext.SaveChanges();
- }
- else
- {
- updatetime = new Updatetime();
- updatetime.Id = Guid.NewGuid().ToString();
- updatetime.Sheng = "涓浗鏀块噰缃戝洓宸濈渷";
- updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
- _ccontext.Updatetimes.Add(updatetime);
- _ccontext.SaveChanges();
- }
+ //datenow = DateTime.Now.Date.AddDays(1);
+
}
catch (Exception ex)
{
@@ -174,7 +174,7 @@
//閲囪喘鍏憡
- public static void caigougonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page, string cgfs, string cgfsName)
+ public static async void caigougonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page, string cgfs, string cgfsName)
{
sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -183,17 +183,41 @@
var list11 = new List<AngleSharp.Dom.IElement>();
while (true)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
logg.WriteLog(page.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
try
{
- using (HttpClient client = new HttpClient())
+ HttpClientHandler handler = new HttpClientHandler();
+ handler.CookieContainer = new CookieContainer();
+ using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(60);
+ client.DefaultRequestHeaders.Add("Accept", "*/*");
+ client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
+ client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
//List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
- var res = response.Content.ReadAsStringAsync().Result;
+ string res = "";
+ if (response.IsSuccessStatusCode)
+ {
+ using (var responseStream = await response.Content.ReadAsStreamAsync())
+ {
+ using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
+ {
+ using (var reader = new StreamReader(decompressedStream))
+ {
+ res = await reader.ReadToEndAsync();
+ // 澶勭悊瑙ｅ帇缂╁悗鐨勫搷搴斿唴瀹�
+ }
+ }
+ }
+ }
+ else
+ {
+ // 澶勭悊璇锋眰澶辫触鐨勬儏鍐�
+ }
+ //var res = response.Content.ReadAsStringAsync().Result;
var document = parser.ParseDocument(res);
var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -236,7 +260,7 @@
foreach (var sichuanjieshoudtl1 in lists)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
try
{
sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl();
@@ -265,7 +289,10 @@
}
using (HttpClient clientdtl = new HttpClient())
{
- var zhengfuProjectcount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "02" && x.OpenTenderTime == DateTime.Parse(NoticeTime)));
+ var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+ var zhengfuProjectcount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "02" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend) );
if (zhengfuProjectcount <= 0)
@@ -498,7 +525,7 @@
_ccontext.SaveChanges();
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
}
}
}
@@ -515,6 +542,8 @@
}
}
}
+
+ page += 1;
}
catch (Exception ex)
{
@@ -523,9 +552,10 @@
logg.WriteLog(ex.StackTrace, "涓浗鏀块噰缃戝洓宸濈渷");
logg.WriteLog(ex.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
+ Thread.CurrentThread.Join(1000 * 60 * 5);//闃绘璁惧畾鏃堕棿
}
- page += 1;
+
}
@@ -539,7 +569,7 @@
//鎰忓悜鍏紑
- public static void yixianggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime,string endTime, int page)
+ public static async void yixianggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime,string endTime, int page)
{
sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -548,17 +578,41 @@
var list11 = new List<AngleSharp.Dom.IElement>();
while (true)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
logg.WriteLog(page.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
try
{
- using (HttpClient client = new HttpClient())
+ HttpClientHandler handler = new HttpClientHandler();
+ handler.CookieContainer = new CookieContainer();
+ using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(60);
+ client.DefaultRequestHeaders.Add("Accept", "*/*");
+ client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
+ client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
//List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
- var res = response.Content.ReadAsStringAsync().Result;
+ //var res = response.Content.ReadAsStringAsync().Result;
+ string res = "";
+ if (response.IsSuccessStatusCode)
+ {
+ using (var responseStream = await response.Content.ReadAsStreamAsync())
+ {
+ using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
+ {
+ using (var reader = new StreamReader(decompressedStream))
+ {
+ res = await reader.ReadToEndAsync();
+ // 澶勭悊瑙ｅ帇缂╁悗鐨勫搷搴斿唴瀹�
+ }
+ }
+ }
+ }
+ else
+ {
+ // 澶勭悊璇锋眰澶辫触鐨勬儏鍐�
+ }
var document = parser.ParseDocument(res);
var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -598,7 +652,7 @@
}
foreach (var sichuanjieshoudtl1 in lists)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
try
{
sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl();
@@ -627,7 +681,10 @@
}
using (HttpClient clientdtl = new HttpClient())
{
- var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "01" && x.OpenTenderTime == DateTime.Parse(NoticeTime)));
+ var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "01" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
if (zhengfuProjectCount <= 0)
{
clientdtl.Timeout = TimeSpan.FromSeconds(60);
@@ -672,8 +729,8 @@
{
if (td[i].TextContent.IndexOf("椤圭洰缂栧彿锛�") >= 0)
{
-
- OpenTenderCode = td[i].TextContent.Replace("椤圭洰缂栧彿锛�", "").Replace("/", "").Trim();
+ if (string.IsNullOrEmpty(OpenTenderCode))
+ OpenTenderCode = td[i].TextContent.Replace("椤圭洰缂栧彿锛�", "").Replace("/", "").Trim();
}
if (td[i].TextContent.IndexOf("棰勭畻閲戦锛�") >= 0)
@@ -806,7 +863,7 @@
_ccontext.SaveChanges();
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
}
}
}
@@ -823,6 +880,8 @@
}
}
}
+
+ page += 1;
}
catch (Exception ex)
{
@@ -831,10 +890,11 @@
logg.WriteLog(ex.StackTrace, "涓浗鏀块噰缃戝洓宸濈渷");
logg.WriteLog(ex.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
+ Thread.CurrentThread.Join(1000 * 60 * 5);//闃绘璁惧畾鏃堕棿
}
- page += 1;
+
}
@@ -848,7 +908,7 @@
//缁撴灉鍏憡
- public static void zhongbiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
+ public static async void zhongbiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
{
sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -857,17 +917,41 @@
var list11 = new List<AngleSharp.Dom.IElement>();
while (true)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
logg.WriteLog(page.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
try
{
- using (HttpClient client = new HttpClient())
+ HttpClientHandler handler = new HttpClientHandler();
+ handler.CookieContainer = new CookieContainer();
+ using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(60);
+ client.DefaultRequestHeaders.Add("Accept", "*/*");
+ client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
+ client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
//List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
- var res = response.Content.ReadAsStringAsync().Result;
+ //var res = response.Content.ReadAsStringAsync().Result;
+ string res = "";
+ if (response.IsSuccessStatusCode)
+ {
+ using (var responseStream = await response.Content.ReadAsStreamAsync())
+ {
+ using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
+ {
+ using (var reader = new StreamReader(decompressedStream))
+ {
+ res = await reader.ReadToEndAsync();
+ // 澶勭悊瑙ｅ帇缂╁悗鐨勫搷搴斿唴瀹�
+ }
+ }
+ }
+ }
+ else
+ {
+ // 澶勭悊璇锋眰澶辫触鐨勬儏鍐�
+ }
var document = parser.ParseDocument(res);
var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -908,7 +992,7 @@
foreach (var sichuanjieshoudtl1 in lists)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
try
{
sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl();
@@ -937,7 +1021,10 @@
}
using (HttpClient clientdtl = new HttpClient())
{
- var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "03" && x.OpenTenderTime == DateTime.Parse(NoticeTime)));
+ var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "03" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
if (zhengfuProjectCount <= 0)
{
clientdtl.Timeout = TimeSpan.FromSeconds(60);
@@ -982,8 +1069,8 @@
{
if (td[i].TextContent.IndexOf("椤圭洰缂栧彿锛�") >= 0)
{
-
- OpenTenderCode = td[i].TextContent.Replace("椤圭洰缂栧彿锛�", "").Replace("涓�銆�", "").Trim();
+ if (string.IsNullOrEmpty(OpenTenderCode))
+ OpenTenderCode = td[i].TextContent.Replace("椤圭洰缂栧彿锛�", "").Replace("涓�銆�", "").Trim();
if (OpenTenderCode.IndexOf("锛堟嫑鏍囨枃浠剁紪鍙�") >= 0)
{
@@ -1030,8 +1117,8 @@
{
if (td[i].TextContent.IndexOf("椤圭洰缂栧彿锛�") >= 0)
{
-
- OpenTenderCode = td[i].TextContent.Replace("椤圭洰缂栧彿锛�", "").Replace("涓�銆�", "").Trim();
+ if (string.IsNullOrEmpty(OpenTenderCode))
+ OpenTenderCode = td[i].TextContent.Replace("椤圭洰缂栧彿锛�", "").Replace("涓�銆�", "").Trim();
if (OpenTenderCode.IndexOf("锛堟嫑鏍囨枃浠剁紪鍙�") >= 0)
{
@@ -1169,7 +1256,7 @@
_ccontext.SaveChanges();
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
}
}
}
@@ -1186,6 +1273,7 @@
}
}
}
+ page += 1;
}
catch (Exception ex)
{
@@ -1194,10 +1282,11 @@
logg.WriteLog(ex.StackTrace, "涓浗鏀块噰缃戝洓宸濈渷");
logg.WriteLog(ex.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
+ Thread.CurrentThread.Join(1000 * 60 * 5);//闃绘璁惧畾鏃堕棿
}
- page += 1;
+
}
@@ -1210,7 +1299,7 @@
}
//鏇存鍏憡
- public static void gengzhenggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
+ public static async void gengzhenggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
{
sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -1219,17 +1308,41 @@
var list11 = new List<AngleSharp.Dom.IElement>();
while (true)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
logg.WriteLog(page.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
try
{
- using (HttpClient client = new HttpClient())
+ HttpClientHandler handler = new HttpClientHandler();
+ handler.CookieContainer = new CookieContainer();
+ using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(60);
+ client.DefaultRequestHeaders.Add("Accept", "*/*");
+ client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
+ client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
//List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
- var res = response.Content.ReadAsStringAsync().Result;
+ //var res = response.Content.ReadAsStringAsync().Result;
+ string res = "";
+ if (response.IsSuccessStatusCode)
+ {
+ using (var responseStream = await response.Content.ReadAsStreamAsync())
+ {
+ using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
+ {
+ using (var reader = new StreamReader(decompressedStream))
+ {
+ res = await reader.ReadToEndAsync();
+ // 澶勭悊瑙ｅ帇缂╁悗鐨勫搷搴斿唴瀹�
+ }
+ }
+ }
+ }
+ else
+ {
+ // 澶勭悊璇锋眰澶辫触鐨勬儏鍐�
+ }
var document = parser.ParseDocument(res);
var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -1270,7 +1383,7 @@
foreach (var sichuanjieshoudtl1 in lists)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
try
{
sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl();
@@ -1299,7 +1412,10 @@
}
using (HttpClient clientdtl = new HttpClient())
{
- var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "04" && x.OpenTenderTime == DateTime.Parse(NoticeTime)));
+ var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "04" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
if (zhengfuProjectCount <= 0)
{
clientdtl.Timeout = TimeSpan.FromSeconds(60);
@@ -1344,8 +1460,8 @@
{
if (td[i].TextContent.IndexOf("鍘熷叕鍛婄殑閲囪喘椤圭洰缂栧彿锛�") >= 0)
{
-
- OpenTenderCode = td[i].TextContent.Replace("鍘熷叕鍛婄殑閲囪喘椤圭洰缂栧彿锛�", "").Trim();
+ if (string.IsNullOrEmpty(OpenTenderCode))
+ OpenTenderCode = td[i].TextContent.Replace("鍘熷叕鍛婄殑閲囪喘椤圭洰缂栧彿锛�", "").Trim();
}
if (td[i].TextContent.IndexOf("棰勭畻閲戦锛�") >= 0)
@@ -1477,7 +1593,7 @@
_ccontext.SaveChanges();
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
}
}
}
@@ -1494,6 +1610,8 @@
}
}
}
+
+ page += 1;
}
catch (Exception ex)
{
@@ -1502,10 +1620,11 @@
logg.WriteLog(ex.StackTrace, "涓浗鏀块噰缃戝洓宸濈渷");
logg.WriteLog(ex.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
+ Thread.CurrentThread.Join(1000 * 60 * 5);//闃绘璁惧畾鏃堕棿
}
- page += 1;
+
}
@@ -1519,7 +1638,7 @@
//搴熸爣鍏憡
- public static void feibiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
+ public static async void feibiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page)
{
sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A"));
@@ -1528,17 +1647,41 @@
var list11 = new List<AngleSharp.Dom.IElement>();
while (true)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
logg.WriteLog(page.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page);
try
{
- using (HttpClient client = new HttpClient())
+ HttpClientHandler handler = new HttpClientHandler();
+ handler.CookieContainer = new CookieContainer();
+ using (HttpClient client = new HttpClient(handler))
{
//List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>();
client.Timeout = TimeSpan.FromSeconds(60);
+ client.DefaultRequestHeaders.Add("Accept", "*/*");
+ client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br");
+ client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn");
HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result;
- var res = response.Content.ReadAsStringAsync().Result;
+ //var res = response.Content.ReadAsStringAsync().Result;
+ string res = "";
+ if (response.IsSuccessStatusCode)
+ {
+ using (var responseStream = await response.Content.ReadAsStreamAsync())
+ {
+ using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
+ {
+ using (var reader = new StreamReader(decompressedStream))
+ {
+ res = await reader.ReadToEndAsync();
+ // 澶勭悊瑙ｅ帇缂╁悗鐨勫搷搴斿唴瀹�
+ }
+ }
+ }
+ }
+ else
+ {
+ // 澶勭悊璇锋眰澶辫触鐨勬儏鍐�
+ }
var document = parser.ParseDocument(res);
var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault();
var contentList = sssdfsdfsd.QuerySelector("ul");
@@ -1579,7 +1722,7 @@
foreach (var sichuanjieshoudtl1 in lists)
{
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
try
{
sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl();
@@ -1608,7 +1751,10 @@
}
using (HttpClient clientdtl = new HttpClient())
{
- var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "11" && x.OpenTenderTime == DateTime.Parse(NoticeTime)));
+ var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "11" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
if (zhengfuProjectCount <= 0)
{
clientdtl.Timeout = TimeSpan.FromSeconds(60);
@@ -1653,8 +1799,8 @@
{
if (td[i].TextContent.IndexOf("閲囪喘椤圭洰缂栧彿锛�") >= 0)
{
-
- OpenTenderCode = td[i].TextContent.Replace("閲囪喘椤圭洰缂栧彿锛�", "").Trim();
+ if (string.IsNullOrEmpty(OpenTenderCode))
+ OpenTenderCode = td[i].TextContent.Replace("閲囪喘椤圭洰缂栧彿锛�", "").Trim();
}
if (td[i].TextContent.IndexOf("棰勭畻閲戦锛�") >= 0)
@@ -1787,7 +1933,7 @@
_ccontext.SaveChanges();
- Thread.CurrentThread.Join(1000 * 5);//闃绘璁惧畾鏃堕棿
+ Thread.CurrentThread.Join(1000 * 10);//闃绘璁惧畾鏃堕棿
}
}
}
@@ -1804,6 +1950,7 @@
}
}
}
+ page += 1;
}
catch (Exception ex)
{
@@ -1812,10 +1959,11 @@
logg.WriteLog(ex.StackTrace, "涓浗鏀块噰缃戝洓宸濈渷");
logg.WriteLog(ex.ToString(), "涓浗鏀块噰缃戝洓宸濈渷");
+ Thread.CurrentThread.Join(1000 * 60 * 5);//闃绘璁惧畾鏃堕棿
}
- page += 1;
+
}
--
Gitblit v1.9.1