| | |
| | | using System; |
| | | using System.Collections.Generic; |
| | | using System.Globalization; |
| | | using System.IO; |
| | | using System.IO.Compression; |
| | | using System.Linq; |
| | | using System.Net; |
| | | using System.Net.Http; |
| | | using System.Net.Http.Headers; |
| | | using System.Text; |
| | |
| | | { |
| | | try |
| | | { |
| | | |
| | | |
| | | //获取更新时间 |
| | | string operationStartTime = ""; |
| | | string operationEndTime = ""; |
| | | Updatetime updatetime = _ccontext.Updatetimes.Where(x => x.Sheng == "中国政采网四川省").FirstOrDefault(); |
| | | if (updatetime != null) |
| | | |
| | | logg.WriteLog("开始判断时间", "中国政采网四川省"); |
| | | |
| | | if (DateTime.Now.Hour == 0 ) |
| | | { |
| | | operationStartTime = updatetime.Updatetime1.ToString("yyyy-MM-dd"); |
| | | operationEndTime = updatetime.Updatetime1.AddDays(1).ToString("yyyy-MM-dd"); |
| | | //updatetime.Updatetime1 = DateTime.Now.AddDays(-1); |
| | | //_ccontext.SaveChanges(); |
| | | DateTime datenow = DateTime.Now.Date; |
| | | Updatetime updatetime = _ccontext.Updatetimes.Where(x => x.Sheng == "中国政采网四川省").FirstOrDefault(); |
| | | logg.WriteLog("datenow="+datenow.ToString("yyyy-MM-dd HH:mm:ss"), "中国政采网四川省"); |
| | | operationStartTime = datenow.AddDays(-1).ToString("yyyy-MM-dd"); |
| | | operationEndTime = datenow.AddDays(-1).ToString("yyyy-MM-dd"); |
| | | logg.WriteLog("operationStartTime=" + operationStartTime, "中国政采网四川省"); |
| | | logg.WriteLog("operationEndTime=" + operationEndTime, "中国政采网四川省"); |
| | | var currPage = 1; |
| | | string sichuanpageurll = ""; |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 意向公开 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=9&dbselect=bidx&kw=%E6%84%8F%E5%90%91&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省意向公开 开始获取", "中国政采网四川省"); |
| | | yixianggonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省意向公开 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | |
| | | #region 公开招标 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=1&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 公开招标 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "1", "公开招标"); |
| | | logg.WriteLog("中国政采网四川省采购公告 公开招标 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | |
| | | #region 询价 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=2&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 询价 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "4", "询价"); |
| | | logg.WriteLog("中国政采网四川省采购公告 询价 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 竞争性谈判 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=3&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 竞争性谈判 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "5", "竞争性谈判"); |
| | | logg.WriteLog("中国政采网四川省采购公告 竞争性谈判 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 单一来源 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=4&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 单一来源 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "7", "单一来源"); |
| | | logg.WriteLog("中国政采网四川省采购公告 单一来源 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 邀请招标 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=6&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 邀请招标 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "2", "邀请招标"); |
| | | logg.WriteLog("中国政采网四川省采购公告 邀请招标 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 竞争性磋商 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=10&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 竞争性磋商 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "3", "竞争性磋商"); |
| | | logg.WriteLog("中国政采网四川省采购公告 竞争性磋商 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 中标公告 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=7&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省中标公告 开始获取", "中国政采网四川省"); |
| | | zhongbiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省中标公告 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 成交公告 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=11&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省成交公告 开始获取", "中国政采网四川省"); |
| | | zhongbiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省成交公告 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 更正公告 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=8&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省更正公告 开始获取", "中国政采网四川省"); |
| | | gengzhenggonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省更正公告 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | //Thread.CurrentThread.Join(1000 * 60 * 30);//阻止设定时间 |
| | | #region 废标公告 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=12&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省废标公告 开始获取", "中国政采网四川省"); |
| | | feibiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省废标公告 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | |
| | | |
| | | //保存最后更新时间 |
| | | if (updatetime != null) |
| | | { |
| | | updatetime.Updatetime1 = datenow; |
| | | _ccontext.SaveChanges(); |
| | | } |
| | | else |
| | | { |
| | | updatetime = new Updatetime(); |
| | | updatetime.Id = Guid.NewGuid().ToString(); |
| | | updatetime.Sheng = "中国政采网四川省"; |
| | | updatetime.Updatetime1 = datenow; |
| | | _ccontext.Updatetimes.Add(updatetime); |
| | | _ccontext.SaveChanges(); |
| | | } |
| | | |
| | | Thread.CurrentThread.Join(1000 * 60 * 60);//阻止设定时间 |
| | | } |
| | | else |
| | | { |
| | | operationStartTime = "2021-12-01"; //DateTime.Now.AddYears(-1).ToString("yyyy-MM-dd"); |
| | | operationEndTime = "2022-12-01";//DateTime.Now.ToString("yyyy-MM-dd"); |
| | | } |
| | | var currPage = 1; |
| | | string sichuanpageurll = ""; |
| | | |
| | | #region 意向公开 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=9&dbselect=bidx&kw=%E6%84%8F%E5%90%91&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省意向公开 开始获取", "中国政采网四川省"); |
| | | yixianggonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省意向公开 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | |
| | | |
| | | #region 公开招标 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=1&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 公开招标 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "1", "公开招标"); |
| | | logg.WriteLog("中国政采网四川省采购公告 公开招标 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | #region 询价 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=2&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 询价 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "4", "询价"); |
| | | logg.WriteLog("中国政采网四川省采购公告 询价 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | #region 竞争性谈判 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=3&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 竞争性谈判 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "5", "竞争性谈判"); |
| | | logg.WriteLog("中国政采网四川省采购公告 竞争性谈判 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | #region 单一来源 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=4&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 单一来源 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "7", "单一来源"); |
| | | logg.WriteLog("中国政采网四川省采购公告 单一来源 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | #region 邀请招标 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=6&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 邀请招标 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "2", "邀请招标"); |
| | | logg.WriteLog("中国政采网四川省采购公告 邀请招标 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | #region 竞争性磋商 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=10&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省采购公告 竞争性磋商 开始获取", "中国政采网四川省"); |
| | | caigougonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage, "3", "竞争性磋商"); |
| | | logg.WriteLog("中国政采网四川省采购公告 竞争性磋商 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | |
| | | |
| | | #region 中标公告 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=7&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省中标公告 开始获取", "中国政采网四川省"); |
| | | zhongbiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省中标公告 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | |
| | | #region 成交公告 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=11&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省成交公告 开始获取", "中国政采网四川省"); |
| | | zhongbiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省成交公告 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | |
| | | #region 更正公告 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=8&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省更正公告 开始获取", "中国政采网四川省"); |
| | | gengzhenggonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime, currPage); |
| | | logg.WriteLog("中国政采网四川省更正公告 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | |
| | | #region 废标公告 |
| | | currPage = 1; |
| | | sichuanpageurll = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=12&dbselect=bidx&kw=&start_time=&end_time=&timeType=6&displayZone=%E5%9B%9B%E5%B7%9D&zoneId=51&pppStatus=&agentName="; |
| | | logg.WriteLog("中国政采网四川省废标公告 开始获取", "中国政采网四川省"); |
| | | feibiaogonggao(_ccontext, sichuanpageurll, operationStartTime, operationEndTime , currPage); |
| | | logg.WriteLog("中国政采网四川省废标公告 开始获取结束", "中国政采网四川省"); |
| | | #endregion |
| | | |
| | | |
| | | |
| | | //保存最后更新时间 |
| | | if (updatetime != null) |
| | | { |
| | | updatetime.Updatetime1 = DateTime.Now.AddDays(-1); |
| | | _ccontext.SaveChanges(); |
| | | } |
| | | else |
| | | { |
| | | updatetime = new Updatetime(); |
| | | updatetime.Id = Guid.NewGuid().ToString(); |
| | | updatetime.Sheng = "中国政采网四川省"; |
| | | updatetime.Updatetime1 = DateTime.Now.AddDays(-1); |
| | | _ccontext.Updatetimes.Add(updatetime); |
| | | _ccontext.SaveChanges(); |
| | | } |
| | | //datenow = DateTime.Now.Date.AddDays(1); |
| | | |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | |
| | | |
| | | |
| | | //采购公告 |
| | | public static void caigougonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page, string cgfs, string cgfsName) |
| | | public static async void caigougonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page, string cgfs, string cgfsName) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | var list11 = new List<AngleSharp.Dom.IElement>(); |
| | | while (true) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | logg.WriteLog(page.ToString(), "中国政采网四川省"); |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | using (HttpClient client = new HttpClient()) |
| | | try |
| | | { |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | foreach (var list11111 in list11) |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | var listcount = list22.Count(x=>x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | if (breakable) |
| | | { |
| | | break; |
| | | } |
| | | } |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | // 处理请求失败的情况 |
| | | } |
| | | |
| | | |
| | | |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | try |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | { |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim() ; |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|","").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach(var ssss in sssssss) |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | { |
| | | if (ssss.IndexOf("采购人")>=0) |
| | | foreach (var list11111 in list11) |
| | | { |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | } |
| | | } |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | if (breakable) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | break; |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | } |
| | | |
| | | |
| | | |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | try |
| | | { |
| | | ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == aaaaaaaa.pageurl).FirstOrDefault(); |
| | | if (zhengfuProject == null) |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | { |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList() ; |
| | | if(fujians!=null&& fujians.Count > 0) |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | { |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">"+ sssss + "</a><br></td></tr>"; |
| | | } |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | content += fujianhtml; |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length>0) |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | { |
| | | var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd"); |
| | | var notimestart = DateTime.Parse(notime); |
| | | var notimeend = notimestart.AddDays(1); |
| | | var zhengfuProjectcount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "02" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend) ); |
| | | |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | if (zhengfuProjectcount <= 0) |
| | | { |
| | | clientdtl.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | { |
| | | if (td[i].TextContent.IndexOf("项目编号:") >= 0 && td[i].TextContent.IndexOf("、") < 0) |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | if (string.IsNullOrEmpty(OpenTenderCode)) |
| | | { |
| | | OpenTenderCode = td[i].TextContent.Replace("项目编号:", "").Replace("1.", "").Replace("/", "").Trim(); |
| | | } |
| | | |
| | | |
| | | |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | if(td[i].TextContent.IndexOf("万元") >= 0) |
| | | content += fujianhtml; |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | { |
| | | |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | { |
| | | if (td[i].TextContent.IndexOf("项目编号:") >= 0 && td[i].TextContent.IndexOf("、") < 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("3.", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | if (string.IsNullOrEmpty(OpenTenderCode)) |
| | | { |
| | | Budget = a * 10000; |
| | | OpenTenderCode = td[i].TextContent.Replace("项目编号:", "").Replace("1.", "").Replace("/", "").Trim(); |
| | | } |
| | | |
| | | |
| | | |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | |
| | | if (td[i].TextContent.IndexOf("万元") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("3.", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a * 10000; |
| | | } |
| | | |
| | | |
| | | } |
| | | else |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("3.", "").Replace("(", "").Replace(")", "").Replace("元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a; |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("时", ":").Replace("分(", "").Replace("分", ":").Replace("秒", "").Replace("(", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | else |
| | | if (td[i].TextContent.IndexOf("1.时间:") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("3.", "").Replace("(", "").Replace(")", "").Replace("元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | if (!OpenTenderTime.HasValue) |
| | | { |
| | | Budget = a; |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("时", ":").Replace("分(", "").Replace("分", ":").Replace("秒", "").Replace("(", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | var sss = td[i].TextContent.Replace("1.时间:", "").Replace("北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("时", ":").Replace("分(", "").Replace("分", ":").Replace("秒", "").Replace("(", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | if (td[i].TextContent.IndexOf("1.时间:") >= 0) |
| | | { |
| | | if (!OpenTenderTime.HasValue) |
| | | { |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("1.时间:", "").Replace("北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("时", ":").Replace("分(", "").Replace("分", ":").Replace("秒", "").Replace("(", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.StartsWith("时间:")) |
| | | { |
| | | if (!OpenTenderTime.HasValue) |
| | | { |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("时间:", "").Replace("北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("时", ":").Replace("分(", "").Replace("分", ":").Replace("秒", "").Replace("(", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | } |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.StartsWith("时间:")) |
| | | { |
| | | if (!OpenTenderTime.HasValue) |
| | | { |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("时间:", "").Replace("北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("时", ":").Replace("分(", "").Replace("分", ":").Replace("秒", "").Replace("(", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | } |
| | | |
| | | } |
| | | } |
| | | |
| | | var zhengfuProject1 = _ccontext.ZhengfuProjects.Where(x => x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "02" && x.OpenTenderCode == OpenTenderCode).FirstOrDefault(); |
| | | if (zhengfuProject1 != null) |
| | | continue; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "02"; |
| | | string gglxName = "采购公告"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null ; |
| | | //CityName = CityName; |
| | | |
| | | var zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "02"; |
| | | string gglxName = "采购公告"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | |
| | | zhengfuProject.Cgfs = cgfs; |
| | | zhengfuProject.Cgfs = cgfs; |
| | | |
| | | |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | |
| | | |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | |
| | | |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = cgfsName; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = cgfsName; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | } |
| | | |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | } |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | |
| | | page += 1; |
| | | } |
| | | page += 1; |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | Thread.CurrentThread.Join(1000 * 60 * 5);//阻止设定时间 |
| | | } |
| | | |
| | | |
| | | |
| | | } |
| | | |
| | |
| | | |
| | | |
| | | //意向公开 |
| | | public static void yixianggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime,string endTime, int page) |
| | | public static async void yixianggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime,string endTime, int page) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | var list11 = new List<AngleSharp.Dom.IElement>(); |
| | | while (true) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | logg.WriteLog(page.ToString(), "中国政采网四川省"); |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | using (HttpClient client = new HttpClient()) |
| | | try |
| | | { |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | foreach (var list11111 in list11) |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | if (breakable) |
| | | { |
| | | break; |
| | | } |
| | | } |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | // 处理请求失败的情况 |
| | | } |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | try |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | { |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | { |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | foreach (var list11111 in list11) |
| | | { |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | } |
| | | } |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | if (breakable) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | break; |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | } |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | try |
| | | { |
| | | ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == aaaaaaaa.pageurl).FirstOrDefault(); |
| | | if (zhengfuProject == null) |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | { |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | { |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | content += fujianhtml; |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | { |
| | | var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd"); |
| | | var notimestart = DateTime.Parse(notime); |
| | | var notimeend = notimestart.AddDays(1); |
| | | var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "01" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); |
| | | if (zhengfuProjectCount <= 0) |
| | | { |
| | | clientdtl.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | { |
| | | if (td[i].TextContent.IndexOf("项目编号:") >= 0) |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | |
| | | OpenTenderCode = td[i].TextContent.Replace("项目编号:", "").Replace("/","").Trim(); |
| | | |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | content += fujianhtml; |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | { |
| | | |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | if (td[i].TextContent.IndexOf("项目编号:") >= 0) |
| | | { |
| | | Budget = a * 10000; |
| | | if (string.IsNullOrEmpty(OpenTenderCode)) |
| | | OpenTenderCode = td[i].TextContent.Replace("项目编号:", "").Replace("/", "").Trim(); |
| | | |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a * 10000; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | OpenTenderTime = a; |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | } |
| | | |
| | | var zhengfuProject1 = _ccontext.ZhengfuProjects.Where(x => x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "01" && x.OpenTenderCode == OpenTenderCode).FirstOrDefault(); |
| | | if (zhengfuProject1 != null) |
| | | continue; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "01"; |
| | | string gglxName = "意向公开"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | |
| | | zhengfuProject.Cgfs = null; |
| | | var zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "01"; |
| | | string gglxName = "意向公开"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | zhengfuProject.Cgfs = null; |
| | | |
| | | |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | |
| | | |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | |
| | | |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = null; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = null; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | } |
| | | |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | } |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | |
| | | page += 1; |
| | | } |
| | | page += 1; |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | Thread.CurrentThread.Join(1000 * 60 * 5);//阻止设定时间 |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | } |
| | | |
| | |
| | | |
| | | |
| | | //结果公告 |
| | | public static void zhongbiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | public static async void zhongbiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | var list11 = new List<AngleSharp.Dom.IElement>(); |
| | | while (true) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | logg.WriteLog(page.ToString(), "中国政采网四川省"); |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | using (HttpClient client = new HttpClient()) |
| | | try |
| | | { |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | foreach (var list11111 in list11) |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | if (breakable) |
| | | { |
| | | break; |
| | | } |
| | | } |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | // 处理请求失败的情况 |
| | | } |
| | | |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | try |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | { |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | { |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | foreach (var list11111 in list11) |
| | | { |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | } |
| | | } |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | if (breakable) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | break; |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | } |
| | | |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | try |
| | | { |
| | | ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == aaaaaaaa.pageurl).FirstOrDefault(); |
| | | if (zhengfuProject == null) |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | { |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | { |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | content += fujianhtml; |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | { |
| | | |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | { |
| | | if (td[i].TextContent.IndexOf("项目编号:") >= 0) |
| | | { |
| | | |
| | | OpenTenderCode = td[i].TextContent.Replace("项目编号:", "").Replace("一、", "").Trim(); |
| | | |
| | | if (OpenTenderCode.IndexOf("(招标文件编号")>=0) |
| | | { |
| | | OpenTenderCode.Substring(0, OpenTenderCode.IndexOf("(招标文件编号")+1).Replace("(", ""); |
| | | } |
| | | |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a * 10000; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | |
| | | |
| | | td = dtl.QuerySelectorAll("h4"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | { |
| | | var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd"); |
| | | var notimestart = DateTime.Parse(notime); |
| | | var notimeend = notimestart.AddDays(1); |
| | | var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "03" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); |
| | | if (zhengfuProjectCount <= 0) |
| | | { |
| | | clientdtl.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | { |
| | | if (td[i].TextContent.IndexOf("项目编号:") >= 0) |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | |
| | | OpenTenderCode = td[i].TextContent.Replace("项目编号:", "").Replace("一、", "").Trim(); |
| | | |
| | | if (OpenTenderCode.IndexOf("(招标文件编号") >= 0) |
| | | { |
| | | OpenTenderCode.Substring(0, OpenTenderCode.IndexOf("(招标文件编号") + 1).Replace("(", ""); |
| | | } |
| | | |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a * 10000; |
| | | } |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | } |
| | | |
| | | var zhengfuProject1 = _ccontext.ZhengfuProjects.Where(x => x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "03" && x.OpenTenderCode == OpenTenderCode).FirstOrDefault(); |
| | | if (zhengfuProject1 != null) |
| | | continue; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "03"; |
| | | string gglxName = "结果公告"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | |
| | | zhengfuProject.Cgfs = null; |
| | | |
| | | |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | |
| | | |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | |
| | | |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = null; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | content += fujianhtml; |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | { |
| | | |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | { |
| | | if (td[i].TextContent.IndexOf("项目编号:") >= 0) |
| | | { |
| | | if (string.IsNullOrEmpty(OpenTenderCode)) |
| | | OpenTenderCode = td[i].TextContent.Replace("项目编号:", "").Replace("一、", "").Trim(); |
| | | |
| | | if (OpenTenderCode.IndexOf("(招标文件编号") >= 0) |
| | | { |
| | | OpenTenderCode.Substring(0, OpenTenderCode.IndexOf("(招标文件编号") + 1).Replace("(", ""); |
| | | } |
| | | |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a * 10000; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | td = dtl.QuerySelectorAll("h4"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | { |
| | | |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | { |
| | | if (td[i].TextContent.IndexOf("项目编号:") >= 0) |
| | | { |
| | | if (string.IsNullOrEmpty(OpenTenderCode)) |
| | | OpenTenderCode = td[i].TextContent.Replace("项目编号:", "").Replace("一、", "").Trim(); |
| | | |
| | | if (OpenTenderCode.IndexOf("(招标文件编号") >= 0) |
| | | { |
| | | OpenTenderCode.Substring(0, OpenTenderCode.IndexOf("(招标文件编号") + 1).Replace("(", ""); |
| | | } |
| | | |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a * 10000; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | |
| | | |
| | | var zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "03"; |
| | | string gglxName = "结果公告"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | |
| | | zhengfuProject.Cgfs = null; |
| | | |
| | | |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | |
| | | |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | |
| | | |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = null; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | } |
| | | |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | } |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | page += 1; |
| | | } |
| | | page += 1; |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | Thread.CurrentThread.Join(1000 * 60 * 5);//阻止设定时间 |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | } |
| | | |
| | |
| | | } |
| | | |
| | | //更正公告 |
| | | public static void gengzhenggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | public static async void gengzhenggonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | var list11 = new List<AngleSharp.Dom.IElement>(); |
| | | while (true) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | logg.WriteLog(page.ToString(), "中国政采网四川省"); |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | using (HttpClient client = new HttpClient()) |
| | | try |
| | | { |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | foreach (var list11111 in list11) |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | if (breakable) |
| | | { |
| | | break; |
| | | } |
| | | } |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | // 处理请求失败的情况 |
| | | } |
| | | |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | try |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | { |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | { |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | foreach (var list11111 in list11) |
| | | { |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | } |
| | | } |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | if (breakable) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | break; |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | } |
| | | |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | try |
| | | { |
| | | ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == aaaaaaaa.pageurl).FirstOrDefault(); |
| | | if (zhengfuProject == null) |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | { |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | { |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | content += fujianhtml; |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | { |
| | | var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd"); |
| | | var notimestart = DateTime.Parse(notime); |
| | | var notimeend = notimestart.AddDays(1); |
| | | var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "04" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); |
| | | if (zhengfuProjectCount <= 0) |
| | | { |
| | | clientdtl.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | { |
| | | if (td[i].TextContent.IndexOf("原公告的采购项目编号:") >= 0) |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | |
| | | OpenTenderCode = td[i].TextContent.Replace("原公告的采购项目编号:", "").Trim(); |
| | | |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | content += fujianhtml; |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | { |
| | | |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | if (td[i].TextContent.IndexOf("原公告的采购项目编号:") >= 0) |
| | | { |
| | | Budget = a * 10000; |
| | | if (string.IsNullOrEmpty(OpenTenderCode)) |
| | | OpenTenderCode = td[i].TextContent.Replace("原公告的采购项目编号:", "").Trim(); |
| | | |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a * 10000; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | OpenTenderTime = a; |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | } |
| | | |
| | | var zhengfuProject1 = _ccontext.ZhengfuProjects.Where(x => x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "04" && x.OpenTenderCode == OpenTenderCode).FirstOrDefault(); |
| | | if (zhengfuProject1 != null) |
| | | continue; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "04"; |
| | | string gglxName = "更正公告"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | var zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "04"; |
| | | string gglxName = "更正公告"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | |
| | | zhengfuProject.Cgfs = null; |
| | | zhengfuProject.Cgfs = null; |
| | | |
| | | |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | |
| | | |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | |
| | | |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = null; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = null; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | } |
| | | |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | } |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | |
| | | page += 1; |
| | | } |
| | | page += 1; |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | Thread.CurrentThread.Join(1000 * 60 * 5);//阻止设定时间 |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | } |
| | | |
| | |
| | | |
| | | |
| | | //废标公告 |
| | | public static void feibiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | public static async void feibiaogonggao(WebCrawlerContext _ccontext, string sichuanpageurll, string startTime, string endTime , int page) |
| | | { |
| | | |
| | | sichuanpageurll = sichuanpageurll.Replace("start_time=", "start_time=" + startTime.Replace("-", "%3A")); |
| | |
| | | var list11 = new List<AngleSharp.Dom.IElement>(); |
| | | while (true) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | logg.WriteLog(page.ToString(), "中国政采网四川省"); |
| | | string sichuanpageurl2 = sichuanpageurll.Replace("page_index=1", "page_index=" + page); |
| | | using (HttpClient client = new HttpClient()) |
| | | try |
| | | { |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | HttpClientHandler handler = new HttpClientHandler(); |
| | | handler.CookieContainer = new CookieContainer(); |
| | | using (HttpClient client = new HttpClient(handler)) |
| | | { |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | //List<sichuanjieshoudtl> data = new List<sichuanjieshoudtl>(); |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | client.DefaultRequestHeaders.Add("Accept", "*/*"); |
| | | client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); |
| | | client.DefaultRequestHeaders.Add("Host", "search.ccgp.gov.cn"); |
| | | HttpResponseMessage response = client.GetAsync(sichuanpageurl2).Result; |
| | | //var res = response.Content.ReadAsStringAsync().Result; |
| | | string res = ""; |
| | | if (response.IsSuccessStatusCode) |
| | | { |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | using (var responseStream = await response.Content.ReadAsStreamAsync()) |
| | | { |
| | | foreach (var list11111 in list11) |
| | | using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) |
| | | { |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | using (var reader = new StreamReader(decompressedStream)) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | res = await reader.ReadToEndAsync(); |
| | | // 处理解压缩后的响应内容 |
| | | } |
| | | } |
| | | if (breakable) |
| | | { |
| | | break; |
| | | } |
| | | } |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | // 处理请求失败的情况 |
| | | } |
| | | |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | var document = parser.ParseDocument(res); |
| | | var sssdfsdfsd = document.All.Where(m => m.ClassName == "vT-srch-result-list").FirstOrDefault(); |
| | | var contentList = sssdfsdfsd.QuerySelector("ul"); |
| | | if (contentList != null) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | try |
| | | var lists = contentList.QuerySelectorAll("li"); |
| | | if (lists == null || lists.Length == 0) |
| | | { |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | break; |
| | | } |
| | | if (page != 1) |
| | | { |
| | | var list22 = lists.ToList(); |
| | | var breakable = true; |
| | | if (list11.Count == list22.Count) |
| | | { |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | foreach (var list11111 in list11) |
| | | { |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | var listcount = list22.Count(x => x.InnerHtml == list11111.InnerHtml); |
| | | |
| | | if (listcount <= 0) |
| | | { |
| | | breakable = false; |
| | | break; |
| | | } |
| | | } |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | if (breakable) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | break; |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | list11 = list22; |
| | | } |
| | | else |
| | | { |
| | | list11 = lists.ToList(); |
| | | } |
| | | |
| | | foreach (var sichuanjieshoudtl1 in lists) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | try |
| | | { |
| | | ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == aaaaaaaa.pageurl).FirstOrDefault(); |
| | | if (zhengfuProject == null) |
| | | sichuanjieshoudtl aaaaaaaa = new sichuanjieshoudtl(); |
| | | aaaaaaaa.pageurl = sichuanjieshoudtl1.QuerySelector("a").GetAttribute("href"); |
| | | |
| | | logg.WriteLog(aaaaaaaa.pageurl, "中国政采网四川省"); |
| | | aaaaaaaa.title = sichuanjieshoudtl1.QuerySelector("a").TextContent.Replace("\n", "").Trim(); |
| | | aaaaaaaa.shorttitle = aaaaaaaa.title; |
| | | |
| | | |
| | | |
| | | string[] sssssss = sichuanjieshoudtl1.QuerySelector("span").TextContent.Replace("|", "").Split("\n"); |
| | | string Purchaser = null; |
| | | string Agency = null; |
| | | var NoticeTime = sssssss[0].Trim(); |
| | | foreach (var ssss in sssssss) |
| | | { |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | if (ssss.IndexOf("采购人") >= 0) |
| | | { |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | content += fujianhtml; |
| | | Purchaser = ssss.Replace("采购人", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | if (ssss.IndexOf("代理机构") >= 0) |
| | | { |
| | | Agency = ssss.Replace("代理机构", "").Replace(":", "").Replace("/", "").Trim(); |
| | | } |
| | | } |
| | | using (HttpClient clientdtl = new HttpClient()) |
| | | { |
| | | var notime = DateTime.Parse(NoticeTime).ToString("yyyy-MM-dd"); |
| | | var notimestart = DateTime.Parse(notime); |
| | | var notimeend = notimestart.AddDays(1); |
| | | var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == aaaaaaaa.pageurl) || (x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "11" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); |
| | | if (zhengfuProjectCount <= 0) |
| | | { |
| | | clientdtl.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responsedtl = clientdtl.GetAsync(aaaaaaaa.pageurl).Result; |
| | | var resdtl = responsedtl.Content.ReadAsStringAsync().Result; |
| | | var documentdtl = parser.ParseDocument(resdtl); |
| | | |
| | | var dtl = documentdtl.All.Where(m => m.ClassName == "vF_detail_content_container").FirstOrDefault(); |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | var content = dtl.OuterHtml; |
| | | |
| | | var fujian = documentdtl.All.Where(m => m.ClassName == "main").FirstOrDefault(); |
| | | |
| | | var fujians = documentdtl.All.Where(m => m.ClassName == "bizDownload").ToList(); |
| | | if (fujians != null && fujians.Count > 0) |
| | | { |
| | | if (td[i].TextContent.IndexOf("采购项目编号:") >= 0) |
| | | var fujianhtml = "<div class=\"bid_attachtab\"><table width=\"700\" border=\"0\" cellspacing=\"1\" bgcolor=\"#efefef\" style=\"text-align: left; width: 100%; margin: 0px auto;\"><tbody>"; |
| | | foreach (var fujianya in fujians) |
| | | { |
| | | |
| | | OpenTenderCode = td[i].TextContent.Replace("采购项目编号:", "").Trim(); |
| | | |
| | | var ssss = fujianya.Id; |
| | | var sssss = fujianya.InnerHtml; |
| | | fujianhtml += "<tr><td class=\"bid_attachtab_content\">附件下载:<a class=\"bizDownload\" target=\"_blank\" href =\"http://download.ccgp.gov.cn/oss/download?uuid=" + ssss + "\" id=\"0E1723104D34335C527765FF6CD28A\" title=\"点击下载\">" + sssss + "</a><br></td></tr>"; |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | fujianhtml += "</tbody></table></div>"; |
| | | |
| | | content += fujianhtml; |
| | | } |
| | | |
| | | |
| | | |
| | | string OpenTenderCode = null; |
| | | decimal? Budget = null; |
| | | DateTime? OpenTenderTime = null; |
| | | |
| | | var td = dtl.QuerySelectorAll("p"); |
| | | |
| | | if (td != null && td.Length > 0) |
| | | { |
| | | |
| | | |
| | | for (int i = 0; i < td.Length; i++) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | if (td[i].TextContent.IndexOf("采购项目编号:") >= 0) |
| | | { |
| | | Budget = a * 10000; |
| | | if (string.IsNullOrEmpty(OpenTenderCode)) |
| | | OpenTenderCode = td[i].TextContent.Replace("采购项目编号:", "").Trim(); |
| | | |
| | | } |
| | | if (td[i].TextContent.IndexOf("预算金额:") >= 0) |
| | | { |
| | | var yusuan = td[i].TextContent.Replace("预算金额:", "").Replace("(", "").Replace(")", "").Replace("万元", "").Replace("人民币", "").Trim(); |
| | | decimal a = 0; |
| | | if (decimal.TryParse(yusuan, out a)) |
| | | { |
| | | Budget = a * 10000; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | if (td[i].TextContent.IndexOf("开标时间:") >= 0) |
| | | { |
| | | OpenTenderTime = a; |
| | | |
| | | |
| | | DateTime a; |
| | | var sss = td[i].TextContent.Replace("开标时间:", "").Replace("(北京时间)", "").Replace("年", "-").Replace("月", "-").Replace("日", " ").Replace("点", ":").Replace("分", "").Trim(); |
| | | if (DateTime.TryParse(sss, out a)) |
| | | { |
| | | OpenTenderTime = a; |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | } |
| | | |
| | | var zhengfuProject1 = _ccontext.ZhengfuProjects.Where(x => x.RecStatus == "A" && x.Title == aaaaaaaa.title && x.Gglx == "11" && x.OpenTenderCode == OpenTenderCode).FirstOrDefault(); |
| | | if (zhengfuProject1 != null) |
| | | continue; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "11"; |
| | | string gglxName = "废标公告"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | |
| | | zhengfuProject.Cgfs = null; |
| | | var zhengfuProject = new ZhengfuProject(); |
| | | zhengfuProject.Id = Guid.NewGuid().ToString(); |
| | | zhengfuProject.Gglx = "11"; |
| | | string gglxName = "废标公告"; |
| | | zhengfuProject.NoticeTime = DateTime.Parse(NoticeTime); |
| | | zhengfuProject.OpenTenderTime = OpenTenderTime; |
| | | zhengfuProject.RegionCode = null; |
| | | //regionName = regionName; |
| | | zhengfuProject.Sheng = "510000"; |
| | | string ShengName = "四川省"; |
| | | zhengfuProject.City = null; |
| | | //CityName = CityName; |
| | | |
| | | |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | zhengfuProject.Cgfs = null; |
| | | |
| | | |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | zhengfuProject.OpenTenderCode = OpenTenderCode; |
| | | zhengfuProject.Budget = Budget; |
| | | |
| | | |
| | | zhengfuProject.Title = aaaaaaaa.title; |
| | | zhengfuProject.Shorttitle = aaaaaaaa.shorttitle; |
| | | zhengfuProject.Pageurl = aaaaaaaa.pageurl; |
| | | zhengfuProject.Purchaser = Purchaser; |
| | | |
| | | |
| | | |
| | | |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | zhengfuProject.Agency = Agency; |
| | | zhengfuProject.AgencyCode = null; |
| | | zhengfuProject.Content = content; |
| | | zhengfuProject.RecStatus = "A"; |
| | | zhengfuProject.Creater = "1"; |
| | | zhengfuProject.Createtime = DateTime.Now; |
| | | zhengfuProject.Modifier = "1"; |
| | | zhengfuProject.Modifytime = zhengfuProject.Createtime; |
| | | _ccontext.ZhengfuProjects.Add(zhengfuProject); |
| | | |
| | | |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = null; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; |
| | | string result = string.Empty; |
| | | Uri postUrl = new Uri(url); |
| | | eswebcrawler eswebcrawler1 = new eswebcrawler(); |
| | | eswebcrawler1.Id = zhengfuProject.Id; |
| | | eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; |
| | | eswebcrawler1.sheng = zhengfuProject.Sheng; |
| | | eswebcrawler1.shengName = ShengName; |
| | | eswebcrawler1.city = zhengfuProject.City; |
| | | eswebcrawler1.cityName = null; |
| | | eswebcrawler1.regionCode = zhengfuProject.RegionCode; |
| | | eswebcrawler1.regionName = null; |
| | | eswebcrawler1.cgfs = zhengfuProject.Cgfs; |
| | | eswebcrawler1.cgfsName = null; |
| | | eswebcrawler1.gglx = zhengfuProject.Gglx; |
| | | eswebcrawler1.gglxName = gglxName; |
| | | eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; |
| | | eswebcrawler1.title = zhengfuProject.Title; |
| | | eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; |
| | | eswebcrawler1.pageurl = zhengfuProject.Pageurl; |
| | | eswebcrawler1.pingmu = zhengfuProject.Pingmu; |
| | | eswebcrawler1.pingmuName = ""; |
| | | eswebcrawler1.purchaser = zhengfuProject.Purchaser; |
| | | eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; |
| | | eswebcrawler1.agency = zhengfuProject.Agency; |
| | | eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; |
| | | eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); |
| | | |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | string requestJson = JsonConvert.SerializeObject(eswebcrawler1); |
| | | |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | using (HttpContent httpContent = new StringContent(requestJson)) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); |
| | | using (HttpClient httpClient = new HttpClient()) |
| | | { |
| | | httpClient.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; |
| | | result = responseMessage.Content.ReadAsStringAsync().Result; |
| | | } |
| | | |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | } |
| | | |
| | | Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); |
| | | |
| | | if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") |
| | | { |
| | | |
| | | } |
| | | else |
| | | { |
| | | throw new Exception("保存数据库出错!"); |
| | | } |
| | | |
| | | |
| | | _ccontext.SaveChanges(); |
| | | |
| | | Thread.CurrentThread.Join(1000 * 4);//阻止设定时间 |
| | | } |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | } |
| | | |
| | | |
| | | } |
| | | } |
| | | page += 1; |
| | | } |
| | | page += 1; |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "中国政采网四川省"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "中国政采网四川省"); |
| | | Thread.CurrentThread.Join(1000 * 60 * 5);//阻止设定时间 |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | } |
| | | |